'''
Functions for visualizing data for EDA
'''
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from IPython.display import display, Markdown
from sklearn.preprocessing import MinMaxScaler

# game_data = ['num_comments', 'num_fights', 'attendance', 'len_game', 'corsi_for%',
#              'fenwick_for%', 'faceoff%', 'off_zone_start%', 'pdo', 'total_sog',
#              'total_pp_goals', 'total_goals', 'total_pp_opportunity', 'total_sh_goals',
#              'sog_diff', 'pp_goals_diff', 'goal_diff', 'pp_opportunity_diff', 'sh_goals_diff',
#              'sog_norm_diff', 'pp_goals_norm_diff', 'goal_norm_diff',
#              'pp_opportunity_norm_diff','sh_goals_norm_diff', 'ot/so', 'comeback?', 'win/loss']

game_data = ['num_comments', 'num_fights', 'attendance', 'len_game', 'corsi_for%',
             'fenwick_for%', 'faceoff%', 'off_zone_start%', 'pdo', 'total_sog',
             'total_pp_goals', 'total_goals', 'total_pp_opportunity', 'total_sh_goals',
             'sog_diff', 'pp_goals_diff', 'goal_diff', 'pp_opportunity_diff', 'sh_goals_diff',
             'ot/so', 'comeback?', 'win/loss']

def plot_team_effect_binary_data(data, statistic=None, game_venue='home'):
    '''Plot Means of binary data stats ('win/loss', 'comeback?')
    '''
    if game_venue == 'home':
        home_data = data.groupby(by='home_team').mean()[statistic]
        plt.figure(figsize=(25,10))
        sns.scatterplot(x='home_team', y=statistic, data=home_data.reset_index(drop=False),
                        s=400, marker='s', color='darkblue')
        plt.hlines(y=home_data.mean(), xmin=-1, xmax=30.4, colors='red',
                   label='Population Mean', linewidth=2)

    elif game_venue == 'away':
        away_data = data.groupby(by='away_team').mean()[statistic]
        plt.figure(figsize=(25,10))
        sns.scatterplot(x='away_team', y=statistic, data=away_data.reset_index(drop=False),
                        s=400, marker='s', color='darkblue')
        plt.hlines(y = away_data.mean(), xmin=-1, xmax=30.4, colors='red',
                   label='Population Mean', linewidth=2)

    elif game_venue == 'all':
        home_data = data.groupby(by='home_team').mean()[statistic]
        away_data = data.groupby(by='away_team').mean()[statistic]
        if statistic == 'win/loss':
            away_data = away_data.map(lambda x: 1-x)
        all_data = (home_data + away_data)/2
        all_data.rename_axis('team', inplace=True)
        plt.figure(figsize=(25,10))
        sns.scatterplot(x='team', y=statistic, data=all_data.reset_index(drop=False),
                        s=400, marker='s', color='darkblue')
        plt.hlines(y = all_data.mean(), xmin=-1, xmax=30.4, colors='red',
                   label='Population Mean', linewidth=2)

    title_font = {'size':'22',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20',
                 'weight': 'medium'}
    tick_font = {'size': '16',
                 'weight': 'medium'}
    statistic = statistic.replace('_', ' ').upper()
    plt.title(f'Team Effect on {statistic} in {game_venue.title()} Games', **title_font)
    plt.ylabel(f'{statistic}', **axis_font)
    plt.xlabel('Team', **axis_font)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font);

# def plot_team_effect_binary_data(data, statistic=None, game_venue='home'):
#     '''Plot Means of binary data stats ('win/loss', 'comeback?')
#     '''
#     if game_venue == 'home':
#         home_data = data.groupby(by='home_team').mean()[statistic]
#         plt.figure(figsize=(25,10))
#         sns.scatterplot(x='home_team', y=statistic, data=home_data.reset_index(drop=False),
#                         s=400, marker='s', color='darkblue')
#         plt.hlines(y=home_data.mean(), xmin=-1, xmax=30.4, colors='red',
#                    label='Population Mean', linewidth=2)

#     elif game_venue == 'away':
#         away_data = data.groupby(by='away_team').mean()[statistic]
#         plt.figure(figsize=(25,10))
#         sns.scatterplot(x='away_team', y=statistic, data=away_data.reset_index(drop=False),
#                         s=400, marker='s', color='darkblue')
#         plt.hlines(y = away_data.mean(), xmin=-1, xmax=30.4, colors='red',
#                    label='Population Mean', linewidth=2)

#     title_font = {'size':'22',
#                   'color':'black',
#                   'weight':'medium',
#                   'verticalalignment':'bottom'}
#     axis_font = {'size':'20',
#                  'weight': 'medium'}
#     tick_font = {'size': '16',
#                  'weight': 'medium'}
#     statistic = statistic.replace('_', ' ').upper()
#     plt.title(f'Team Effect on {statistic} in {game_venue.title()} Games', **title_font)
#     plt.ylabel(f'{statistic}', **axis_font)
#     plt.xlabel('Team', **axis_font)
#     plt.xticks(**tick_font)
#     plt.yticks(**tick_font);

def plot_team_effect(data, statistic=None, game_venue='all', kind='boxplot'):
    '''Plot distribution of game statistics in which a given team plays

    statistic = string of game statistic (e.g total_goals, sog_diff). Default = None.

    game_venue = 'home' or 'away' or 'all'. Selects venue to show game stats from (i.e.
    'home' shows game stats for all home games of a given team)

    kind = 'boxplot' or 'violinplot' or 'stripplot'. Selects which type of distribution
    plot to display.
    '''

    if game_venue == 'all':
        teams = sorted(list(data.home_team.unique()))
        team_data = dict()

        for team in teams:
            one_team_data = data.loc[(data.home_team == team) | (data.away_team == team)].reset_index(drop=True)
            team_data[team] = one_team_data[statistic]

        team_data = pd.DataFrame(team_data)

        if kind == 'boxplot':
            plt.figure(figsize=(25,10))
            sns.boxplot(data=team_data)
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                        label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        elif kind == 'violinplot':
            plt.figure(figsize=(25,10))
            sns.violinplot(data=team_data)
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                        label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        elif kind == 'stripplot':
            plt.figure(figsize=(25,10))
            sns.stripplot(data=team_data, jitter=0.35)
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                        label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        else:
            return KeyError('Invalid Plot Kind')

    elif game_venue == 'home' or 'away':

        if kind == 'boxplot':
            plt.figure(figsize=(25,10))
            sns.boxplot(x=f'{game_venue}_team', y=statistic,
                        data=data.sort_values(by=f'{game_venue}_team'))
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                            label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        elif kind == 'violinplot':
            plt.figure(figsize=(25,10))
            sns.violinplot(x=f'{game_venue}_team', y=statistic,
                           data=data.sort_values(by=f'{game_venue}_team'))
            plt.hlines(y=data[statistic].mean(), xmin=-1, xmax=30.4, colors='red',
                            label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        elif kind == 'stripplot':
            plt.figure(figsize=(25,10))
            sns.stripplot(x=f'{game_venue}_team', y=statistic, jitter=0.35,
                         data=data.sort_values(by=f'{game_venue}_team'))
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                            label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        else:
            return KeyError('Invalid Plot Kind')

    else:
        return KeyError('Invalid game_venue')

    title_font = {'size':'22',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20',
                 'weight': 'medium'}
    tick_font = {'size': '16',
                 'weight': 'medium'}
    sns.despine(offset=0, trim=True)
    statistic = statistic.replace('_', ' ').upper()
    plt.title(f'Team Effect on {statistic} in {game_venue.title()} Games', **title_font)
    plt.ylabel(f'{statistic}', **axis_font, labelpad=5)
    plt.xlabel('Team', **axis_font, labelpad=5)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font);

def plot_team_specific_stats(data, statistic=None, against=False):
    '''Plot distribution of team-specific statistics

    statistic = string of statistic that is available for both home and away (e.g corsi,
    goals, sog, etc). Default = None.

    against = Boolean. False will show distribution of statistic for a given team, True will
    show distribution of statistic against a given team. Default = False.
    '''
    teams = list(data.home_team.unique())
    team_data = dict()

    for team in teams:
        if against == False:
            home_statistic = data.loc[(data.home_team == team)]['home_' + statistic]
            away_statistic = data.loc[(data.away_team == team)]['away_' + statistic]
        elif against == True:
            home_statistic = data.loc[(data.home_team == team)]['away_' + statistic]
            away_statistic = data.loc[(data.away_team == team)]['home_' + statistic]

        statistic_data = pd.concat([home_statistic, away_statistic], ignore_index=True)
        team_data[team] = statistic_data

    team_data = pd.DataFrame(team_data)
    plt.figure(figsize=(25,10))
    sns.boxplot(data=team_data)
    population_stat_data = (list(data['home_' + statistic]) +
                            (list(data['away_' + statistic])))
    plt.hlines(y=np.median(population_stat_data), xmin=-1, xmax=30.4,
               colors='red', label='Population Median', linewidth=2)
    plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)
    title_font = {'size':'22',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20', 'weight': 'medium'}
    tick_font = {'size': '16', 'weight': 'medium'}
    sns.despine(offset=0, trim=True)

    if against == False:
        statistic = statistic + '_for'
    elif against == True:
        statistic = statistic + '_against'

    statistic = statistic.replace('_', ' ').upper()
    plt.title(f'Team Effect on {statistic} in Games', **title_font)
    plt.ylabel(f'{statistic}', **axis_font, labelpad=5)
    plt.xlabel('Team', **axis_font, labelpad=5)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font);

def plot_numeric_features_dist(data, kind='boxplot', size=(12,8), normalize=True):
    '''
    Plot distributions of numeric features within a DataFrame

    size = tuple with desired size of output figure (width, height)

    normalize = Boolean (Default = True). If True, numerical data is normalized using
    sklearn.preprocessing.MinMaxScaler(). If False, numerical data is not altered.
    '''
    columns = []
    for column in data.columns:
        if (data[column].dtype != 'O'):
            columns.append(column)
    if normalize == True:
        scaler = MinMaxScaler()
        scaled_data = scaler.fit_transform(data[columns])
        data = pd.DataFrame(scaled_data, columns=columns)
    if kind == 'boxplot':
        plt.figure(figsize=(size))
        sns.boxplot(data=data[columns])
    elif kind == 'violinplot':
        plt.figure(figsize=(size))
        sns.violinplot(data=data[columns], scale='count', cut=0)
    elif kind == 'stripplot':
        plt.figure(figsize=(size))
        sns.stripplot(data=data[columns], jitter=0.25)
    title_font = {'size':'18',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    tick_font = {'size': '12',
                 'weight': 'medium'}
    if normalize == True:
        plt.title(f'Normalized Distribution of Numeric Features', **title_font)
    elif normalize == False:
        plt.title(f'Distribution of Numeric Features', **title_font)
    formatted_x_ticks = [x.replace('_', ' ').replace(' polarity', '').title() for x in columns]
    plt.yticks(**tick_font)
    plt.xticks(ticks=plt.xticks()[0], labels=formatted_x_ticks, rotation=60, ha='right', **tick_font);

def plot_game_binary_data(data, size=(16, 5)):
    '''
    Plot distributions of binary/ordinal game data
    '''

    title_font = {'size':'16',
              'color':'black',
              'weight':'medium',
              'verticalalignment':'bottom'}
    axis_font = {'size':'14', 'weight': 'medium'}
    tick_font = {'size': '16', 'weight': 'medium'}

    fig, axs = plt.subplots(1, len(data.columns), figsize=size)
    for i, column in enumerate(data.columns):
        axs[i].hist(data[column])
        column = column.upper()
        axs[i].set_title(column, **title_font)
    axs[0].set_ylabel('Frequency', **axis_font);

def plot_team_effect_comment_fandom(comment_data, size=(12,6)):
    '''
    Plots percentage of comments from a team fanbase that are in game threads that their respective team played in

    comment_data = output of clean_data.extract_comment_data()
    '''
    nhl_abbreviations = ['CBJ', 'TBL', 'MTL', 'FLA', 'NSH', 'SJS', 'LAK', 'BOS', 'COL', 'OTT', 'DAL', 'CGY', 'MIN',
                    'NYI', 'NYR', 'BUF', 'WSH', 'PIT', 'WPG', 'VAN', 'PHI', 'CHI', 'CAR', 'ANA', 'EDM', 'STL',
                    'DET', 'TOR', 'NJD', 'ARI', 'VGK']

    title_font = {'size':'20',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'16', 'weight': 'medium'}
    tick_font = {'size': '14', 'weight': 'medium'}

    plt.figure(figsize=size)
    comment_data.groupby('flair').mean().loc[nhl_abbreviations]['fan'].plot(kind='bar')
    plt.hlines(y=np.mean(comment_data.groupby('flair').mean().loc[nhl_abbreviations]['fan']), xmin=0, xmax=31, color='red')
    plt.title(label='Percentage of Fan Comments by Team', **title_font)
    plt.xlabel('Team', **axis_font)
    plt.ylabel('Percetage of Fan Comments', **axis_font)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font);

def plot_scoring_dist(comment_data, scoring=None, size=(10, 6)):
    '''
    Plot distributions of binary/ordinal game data
    '''
    perc_neutral = np.round(len(comment_data.loc[comment_data[scoring] == 0])/len(comment_data),
                            decimals=3)
    title_font = {'size':'16',
              'color':'black',
              'weight':'medium',
              'verticalalignment':'bottom'}
    axis_font = {'size':'14', 'weight': 'medium'}
    tick_font = {'size': '12', 'weight': 'medium'}

    plt.figure(figsize=size)
    comment_data[scoring].plot(kind='hist', bins=100)
    plt.ylabel('Frequency', **axis_font)
    scoring = scoring.replace('_', ' ').title()
    plt.xlabel(f'{scoring}', **axis_font)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font)
    plt.show();

    print(f'Percentage of neutral comments: {perc_neutral}')

def plot_nlp_heatmap(thread_data, size=(14,17)):
    '''
    Plot heatmap showing correlation between sentiment scores and game features
    '''
    nlp_data = ['fan%', 'vader_comment',
       'socialsent_comment', 'vader_context',
       'socialsent_context', 'positive_context', 'non-fan_context',
       'no_ref_context']

    corr_data = thread_data[game_data + nlp_data].corr().applymap(lambda x: np.round(x, 2))
    corr_data = corr_data.drop(index=nlp_data)[nlp_data]

    color_bar = {
        'shrink': 0.5,
        'pad': 0.01
    }

    title_font = {'size':'20',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20',
                 'weight': 'medium'}
    tick_font = {'size': '12',
                 'weight': 'medium'}

    plt.figure(figsize=size)
    sns.heatmap(corr_data, annot=True, linewidths=0.1,
                square=False, cbar=True, cbar_kws=color_bar,
               vmin=-1, vmax=1, cmap="RdBu_r");
    plt.xticks(rotation=30, ha='right', **tick_font)
    plt.yticks(rotation=0, ha='right', **tick_font)

    plt.title('Correlation of Game Features with Sentiment Scores', **title_font);

def plot_game_feature_heatmap(total_data):
    '''
    Plot showing correlation between game features
    '''
    corr_data = total_data[game_data].corr().applymap(lambda x: np.round(x, 2))

    mask = np.zeros_like(corr_data)
    mask[np.triu_indices_from(mask)] = True

    color_bar = {
        'shrink': 0.5,
        'pad': 0.01
    }

    title_font = {'size':'20',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20',
                 'weight': 'medium'}
    tick_font = {'size': '12',
                 'weight': 'medium'}

    plt.figure(figsize=(17,17))
    sns.heatmap(corr_data, annot=True, linewidths=0.1,
                square=False, cbar=True, cbar_kws=color_bar,
               vmin=-1, vmax=1, cmap="RdBu_r", mask=mask);
    plt.xticks(rotation=90, ha='right', **tick_font)
    plt.yticks(rotation=0, ha='right', **tick_font)

    plt.title('Correlation of Game Features', **title_font);


import re, json
from flask import Flask, request

app = Flask(__name__)

@app.route("/getPersonCardInfo")
def getPersonCardInfo():
    cardno = request.args.get("cardno", "")
    card_info = get_cardno_info(cardno)
    return json.dumps(card_info, ensure_ascii=False)

def get_cardno_info(cardno):
    ret = {"err": 1, "area": "", "birthday": "", "sex": ""}

    cardno = cardno.upper()
    if re.fullmatch("[1-6]\\d{16}[\\d|X]", cardno) == None:
        return ret

    code = (7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2)
    i, s = 0, 0
    while i < 17:
        s += int(cardno[i]) * code[i]
        i += 1

    s = (12 - (s % 11)) % 11

    if s == 10:
        s = "X"
    else:
        s = str(s)

    if s != cardno[17]:
        return ret

    # 提取身份证号中的信息
    area_code = cardno[:6]
    with open("pid_data.txt", "r", encoding="utf-8") as f:
        while True:
            line = f.readline()
            if line == "":
                break

            if line[:6] == area_code:
                ret["area"] = line.split()[1]
                break

    birthday = cardno[6:14]
    birthday = "%s年%d月%d日" % (birthday[0:4], int(birthday[4:6]), int(birthday[6:8]))
    ret["birthday"] = birthday
    sex = "男"
    if int(cardno[-2]) % 2 == 0:
        sex = "女"
    ret["sex"] = sex
    if ret["area"] != "" and ret["birthday"] != "" and ret["sex"] != "":
        ret["err"] = 0
    return ret

if __name__ == '__main__':
    app.run(debug=True)

获取身份证信息接口.py

获取身份证信息接口.py


from app.common.util.json_util import to_json
from flask import Response
import app.auth.constant as constant

MIME_TYPE = 'application/json;charset=utf-8'
DEFAULT_MESSAGE = ''

class Resp:

    def __init__(self):
        pass

    @classmethod
    def success(cls, message=constant.SUCCESS_MESSAGE, data=None):
        """请求成功结果

        :param message: 消息
        :param data: 数据
        """
        resp = cls()
        resp.message = message
        if data:
            resp.data = data
        return resp

    @classmethod
    def error(cls, status=constant.INTERNAL_ERROR, description=None, fields=None):
        """请求失败结果

        :param status: 错误状态码
        :param description: 用户可读的错误消息
        :param fields: 字段描述，通常用于描述请求参数的错误
        """
        resp = cls()
        _error = cls()
        _error.status = status
        if description:
            _error.description = description
        if fields:
            _error.fields = fields
        resp.error = _error
        return resp

    def to_json(self):
        return to_json(self)

def success(message=constant.SUCCESS_MESSAGE, data=None):
    """请求成功结果

    :param message: 消息
    :param data: 数据
    """
    return Response(Resp.success(message=message, data=data).to_json(), mimetype=MIME_TYPE)

def error(status=None, description=None, fields=None, http_status=None):
    """请求失败结果

    :param status: 错误状态码
    :param description: 用户可读的错误消息
    :param fields: 字段描述，通常用于描述请求参数的错误
    :param http_status: http状态码
    """
    resp = Response(Resp.error(status=status, description=description, fields=fields).to_json(), mimetype=MIME_TYPE)
    if http_status:
        resp.status_code = http_status
    return resp


#! /usr/bin/env python3

import matplotlib.pyplot as plt
import pandas as pd
pd.set_option('display.max_columns', 6)

mismatch = 2
df = pd.read_table(f'Result-m{mismatch}.tsv')

df.columns = ["target", "query", "start", "end", "orientation", "edits"]

df = df.astype({'edits': 'int'})
df = df.astype({'start': 'int'})
df = df.astype({'end': 'int'})

df['queryName'] = df['query'].str.split('_').str[1]
df['querySeq'] = df['query'].str.split('_').str[0]
df['targetName'] = df['target'].str.split('_').str[1]
df['targetSeq'] = df['target'].str.split('_').str[0]
df['family'] = ""

def add_name_to_query(dataf, query_sequence, counter):
    query_slice = dataf.loc[dataf["querySeq"] == query_sequence]
    for query_row in query_slice.itertuples():
        if query_row.family == "":
            dataf.loc[query_row.Index, "family"] = counter
            target_sequence = query_row.targetSeq
            dataf = add_name_to_query(dataf, target_sequence, counter)
    return dataf

novel = len(df[df["queryName"] == "novel"]["querySeq"].unique())

family = 0

# Get all sequences of novel miRNAs
for querySequence in df[df["queryName"] == "novel"]["querySeq"].unique():
    for mapping in df[df["querySeq"] == querySequence].itertuples():
        targetSequence = mapping.targetSeq
        if mapping.family == "":
            df.loc[df["querySeq"] == querySequence, "family"] = family
            df = add_name_to_query(df, targetSequence, family)
            print(family)
    ## Null + current count(0 indexed) + 1 -->family=0; unique={0,1}; len=2 means that a family was added in run
    if len(df["family"].unique()) > family+1:
        family += 1
print(df['accession'])

### Possible new stuff

piv = pd.pivot_table(df, "query", "accession", "queryName", aggfunc="count")['novel']
piv = pd.pivot_table(df, "queryName", "family", aggfunc="count")

## To make fig
dfNoDup = df[["query", "queryName", "querySeq", "family"]].drop_duplicates()
piv = pd.pivot_table(dfNoDup, "queryName", "family", aggfunc="count")

y = list(piv.values)
y.pop()
x = list(piv.index)
x.pop()
plt.title(f'Novel miRNAs that have been grouped\n into the same accession with mismatch=2')
plt.xlabel("accessions")
plt.ylabel("# mature miRNA")
plt.plot(x, y)
plt.savefig("Novel count same accession.png")

# Multiple conditionals
# df.loc[(df["accession"] != "") & (df["queryName"]=="novel"),:]

df2 = pd.read_table("~/Downloads/TEST_miRNAPlantPortal/all_seq-sRNA-SuLop.tsv")
df2.index = df2['sequence']

# 31372
dfNovelAll = df2[df2['name'] == "novel"]

# Import Excel
# pip install openpyxl
xlsx = pd.read_excel("~/Documents/Lab-People/Susana Lopes/Artigo sRNA/Supplementary Tables.xlsx",sheet_name=2,header=3)

# Exclusion attempt
expCols = [lib for lib in df2.columns if lib.startswith("Lib")]

labels = [["meta"] * 2 + ["abs"] * len(expCols) + ["bool"] * len(expCols), ["sequence", "name"] + expCols * 2]
tuples = list(zip(*labels))
cols = pd.MultiIndex.from_tuples(tuples, names=["Type", "Name"])

for lib in expCols:
    dfNovelAll.loc[lib+"_bool"] = 0
    dfNovelAll.loc[dfNovelAll[lib] > 0, lib+"_bool"] = 1

dfNovelAll.columns = cols

dfNovelAll.loc[:, 'mapped'] = False
dfNovelAll.loc[dfNovelAll['sequence'].isin(df.querySeq.unique()), 'mapped'] = True
dfNovelAll.loc[:, 'susana'] = False
susanaSeq = xlsx[xlsx.names == "novel"].Sequence.unique()
dfNovelAll.loc[dfNovelAll.sequence.isin(susanaSeq), "susana"] = True

# pip install matplotlib-venn
from matplotlib_venn import venn2

venn2([set(dfNovelAll[dfNovelAll.mapped].index), set(dfNovelAll[dfNovelAll.susana].index)], ("Mapped", "Susana"))
plt.show()

from matplotlib_venn import venn3

venn3([set(dfNovelAll.index), set(dfNovelAll[dfNovelAll.mapped].index), set(dfNovelAll[dfNovelAll.susana].index)], ("All", "Mapped", "Susana"))
plt.title("Novel miRNAs Qsuber dataset")
plt.savefig("Venn diagram all novel Susana.png")

dfNovelAll.loc[:, "family"] = None

mapped = df.loc[df.queryName == "novel", ['querySeq', "family"]].drop_duplicates()

for row in mapped.itertuples():
    dfNovelAll.loc[row.querySeq, 'family'] = row.family

novelCounter = df[df.family != ""].family.astype("int").max()
for row in dfNovelAll.itertuples():
    if row.family is None:
        novelCounter += 1
        dfNovelAll.loc[row.sequence, 'family'] = novelCounter

#From 31372 to 25077
dfNovelAll.to_excel("Novel_with_family.xlsx")


"""
This module will set basic configuration for loggers.
"""

import logging
from typing import Any

formatter = logging.Formatter(
    "%(asctime)s %(levelname)s - LogGenerationFile: %(module)s - Line Number: %(lineno)d - Log:%(message)s"
)

def setup_logger(name: str, log_file: str, level: Any = logging.INFO) -> Any:
    """To setup as many loggers as you want"""

    handler = logging.FileHandler(log_file, mode="a")
    handler.setFormatter(formatter)

    logger = logging.getLogger(name)
    logger.setLevel(level)
    logger.addHandler(handler)

    return logger


import json
import math
from pathlib import Path
from typing import Optional, Dict, List, Any

from flask import Blueprint, Flask, send_from_directory, url_for, request, render_template, flash, send_file
from werkzeug.security import safe_join
from werkzeug.utils import secure_filename, redirect

# to consider later: https://github.com/un33k/python-slugify

bp = Blueprint('manga', __name__, url_prefix='/manga')

instance_path: Optional[Path] = None
media_dir_path: Optional[Path] = None

def init(app: Flask):
    global instance_path, media_dir_path
    instance_path = Path(app.instance_path)
    media_dir_path = Path(app.instance_path, 'media')

# Root
@bp.get('/')
def root():
    return "This is not the URL you're looking for", 404

# Favicon
@bp.get('/favicon.png')
def favicon():
    favicon_path = Path(instance_path, 'favicon.png')
    if favicon_path.is_file():
        return send_file(path_or_file=favicon_path, mimetype='image/png')
    return '', 404

# CSS
@bp.get('/style.css')
def main_css():
    return send_file(path_or_file=Path(instance_path, 'style.css'), mimetype='text/css')

# Media endpoint for serving static files
@bp.get('/media/<path:filename>', endpoint='media')
def manga_media(filename: str):
    return send_from_directory(directory=media_dir_path, path=filename)

# Rules just for building URLs to media files, the actual routing goes to the media endpoint
bp.add_url_rule('/media/<string:series>/<path:filename>', endpoint='media_series', build_only=True)
bp.add_url_rule('/media/<string:series><string:chapter>/<path:filename>', endpoint='media_chapter', build_only=True)

def load_manga_json(series_slug: str) -> Optional[Dict[str, Any]]:
    json_path = Path(media_dir_path, f'{series_slug}.json')
    if not json_path.is_file():
        return None
    with json_path.open(mode='rt', encoding='utf-8') as f:
        return json.load(f)

# Series page
@bp.get('/<string:series_slug>')
def view_series(series_slug: str):
    series_slug = secure_filename(series_slug)
    page = request.args.get('page', default=1, type=int)

    manga_json = load_manga_json(series_slug=series_slug)
    if manga_json is None:
        return redirect(url_for('.root'))

    title = manga_json['title']
    chapters: List[Dict[str, Any]] = manga_json['chapters']

    # Slice chapters for pagination
    page_size = 20      # constant for now
    page_count = max(1, math.ceil(len(chapters) / page_size))
    page = max(1, min(page_count, page))
    chapters = chapters[page_size * (page - 1):page_size * page]

    return render_template('series_view.html', series_slug=series_slug, series_title=title, page=page,
                           last_page=page_count, chapters=chapters)

# Chapter page
@bp.get('/<string:series_slug>/<string:chapter_slug>')
def view_chapter(series_slug: str, chapter_slug: str):
    series_slug = secure_filename(series_slug)
    chapter_slug = secure_filename(chapter_slug)

    manga_json = load_manga_json(series_slug=series_slug)
    if manga_json is None:
        return redirect(url_for('.root'))

    chapters = manga_json['chapters']
    chapter, ci = next(((c, i) for i, c in enumerate(chapters) if c['slug'] == chapter_slug), (None, None))
    if chapter is None:
        flash('chapter not found', category='error')
        return redirect(url_for('.view_series', series_slug=series_slug))

    prev_chapter_slug = chapters[ci - 1]['slug'] if ci > 0 else None
    next_chapter_slug = chapters[ci + 1]['slug'] if ci < len(chapters) - 1 else None

    return render_template('chapter_view.html', series=manga_json, chapter=chapter, contents=chapter['contents'],
                           prev_slug=prev_chapter_slug, next_slug=next_chapter_slug)


from django.urls import path
from .views import current_user, UserList, StorePracticeData, StoreTestData, RetrieveStatistics

urlpatterns = [
    path('current_user/', current_user),
    path('users/', UserList.as_view()),
    path('store_practice_data/', StorePracticeData.as_view()),
    path('store_test_data/', StoreTestData.as_view()),
    path('fetch_stats/', RetrieveStatistics.as_view()),
]


import pyautogui as pag
import time
import os
import tkinter
import loadsettings
import move
ws = loadsettings.load()["walkspeed"]

def apd(k):
    cmd = """
        osascript -e  'tell application "System Events" to key down "{}"'
    """.format(k)
    os.system(cmd)
def apu(k):
    cmd = """
        osascript -e  'tell application "System Events" to key up "{}"'
    """.format(k)
    os.system(cmd)

def sleep(duration, get_now=time.perf_counter):
    now = get_now()
    end = now + duration
    while now < end:
        now = get_now()

move.hold("w",1)
pag.keyDown("s")
sleep(0.2)
pag.keyUp("s")
move.hold("a",1.5)
move.hold("d",3)


#!/usr/bin/env python
# encoding: utf-8
"""
@author: HuRuiFeng
@file: 035_remove-minimum-characters.py
@time: 2023/8/8 10:18
@project: huawei-od-python
@desc: 035 删除最少的字符
"""
from collections import Counter

def solve_method(chars):
    # 统计字符频率
    freq = Counter(chars)
    # 求最小值
    min_freq = min(freq.values())
    result = ""
    for c in chars:
        if freq[c] != min_freq:
            result += c

    if len(result) == 0:
        return 'empty'
    return result

if __name__ == '__main__':
    assert solve_method("abcdd") == "dd"
    assert solve_method("aabbccdd") == "empty"


import os

import cv2.cv2 as cv2
import numpy as np
from torch.utils.data import Dataset as BaseDataset

class Dataset(BaseDataset):

    CLASSES = ['phyto']

    def __init__(
            self,
            images_dir,
            masks_dir,
            classes=None,
            augmentation=None,
            preprocessing=None,
    ):
        self.ids = os.listdir(images_dir)
        self.ids[0] = self.ids[-1]
        self.images_fps = [os.path.join(images_dir, image_id) for image_id
                           in
                           self.ids]
        self.masks_fps = [os.path.join(masks_dir, image_id) for image_id in
                          self.ids]

        # convert str names to class values on masks
        self.class_values = [self.CLASSES.index(cls.lower()) for cls in
                             classes]

        self.augmentation = augmentation
        self.preprocessing = preprocessing

    def __getitem__(self, i):

        # read data
        image = cv2.imread(self.images_fps[i])
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        mask = cv2.imread(self.masks_fps[i], 0)

        # extract certain classes from mask (e.g. cars)
        masks = [(mask == v) for v in self.class_values]
        mask = np.stack(masks, axis=-1).astype('float')

        if self.augmentation:
            sample = self.augmentation(image=image, mask=mask)
            image, mask = sample['image'], sample['mask']

        # apply preprocessing
        if self.preprocessing:
            sample = self.preprocessing(image=image, mask=mask)
            image, mask = sample['image'], sample['mask']

        return image, mask

    def __len__(self):
        return len(self.ids)


# import libraries
import sys

import pandas as pd
import pickle
from sqlalchemy import create_engine

from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report, f1_score
from sklearn.model_selection import GridSearchCV

from sklearn.base import BaseEstimator, TransformerMixin

from scipy.stats import hmean

import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
from nltk import pos_tag

nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')

import re
import numpy as np

def load_data(database_filepath):
    """
	load data from database

	Input:
	database_filepath : path to database

	Output:
	X, Y, category_names : inputs to ML model
	"""

    database_filepath = "sqlite:///" + database_filepath
    engine = create_engine(database_filepath)
    df = pd.read_sql_table('MessagesTable', engine)
    X = df['message'].values
    Y = df.drop(['id', 'message', 'original', 'genre'], axis=1).values
    category_names = df.columns[3:]

    print('Loaded {} samples'.format(len(df)))

    return X, Y, category_names

def tokenize(text):
    """
	Tokenization function.

	Input:
	text : string to tokenize

	Output:
	tokens : list of tokens
	"""

	# URL processing
    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    detected_urls = re.findall(url_regex, text)
    for url in detected_urls:
        text = text.replace(url, "urlplaceholder")

    # Case Normalization and punctuation removal
    text = re.sub(r"[^a-zA-Z0-9]", " ", text.lower())

    # tokenize text
    tokens = word_tokenize(text)

	# Remove stopwords
    stop_words = stopwords.words("english")
    lemmatizer = WordNetLemmatizer()
    stop_words = list(set(stop_words) - {'over', 'not'})

    # lemmatize and stem
    tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stop_words]
    tokens = [lemmatizer.lemmatize(word, pos='n') for word in tokens]
    tokens = [lemmatizer.lemmatize(word, pos='v') for word in tokens]
    tokens = [PorterStemmer().stem(w) for w in tokens]

    return tokens

def generalized_fscore(y_true, y_pred):
    """
    Scoring function used for gridsearch.
    We use a generalization of the F score for precision recall for the '1' class of each category.

    Input :
    y_true, y_pred : arrays of true values and model predictions

    Output:
    f1score : model score
    """
    score_list = []
    for column in range(y_true.shape[1]):
		# First compute each F score for the '1' class of each category
        score = f1_score(y_true[:,column], y_pred[:,column], average='binary', labels=[1])
		# Add 0.01 to avoid zero scores
        score = min(1, score+0.01)
        score_list.append(score)
    f1_score_vec = np.asarray(score_list)
	# Compute harmonic mean
    f1score = hmean(f1_score_vec)
    return  f1score

class QuestionExtractor(BaseEstimator, TransformerMixin):
    """
	Transformer that returns 'True' if there is a question in the text.
	This feature was proven useful through grid-search.
	"""
    def __init__(self, used=True):
		# 'used' is boolean to decide if the feature is used or not.
        self.used = used

    def question_matcher(self, text):
        if text.find('?') != -1 and self.used:
            return True
        else:
            return False

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        X_tagged = pd.Series(X).apply(self.question_matcher)
        return pd.DataFrame(X_tagged)

class DollarsExtractor(BaseEstimator, TransformerMixin):
    """
	Transformer that returns 'True' if there is an amount in dollars in the text, like $25,000.
	This feature was proven useful through grid-search.
	"""

    def __init__(self, used=True):
		# 'used' is boolean to decide if the feature is used or not.
        self.used = used

    def dollars_matcher(self, text):
        money_regex = r"\$\ ?[+-]?[0-9]{1,3}(?:,?[0-9])*(?:\.[0-9]{1,2})?"
        z = re.findall(money_regex, text)
        if len(z)>0 and self.used:
            return True
        else:
            return False

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        X_tagged = pd.Series(X).apply(self.dollars_matcher)
        return pd.DataFrame(X_tagged)

class NumSentExtractor(BaseEstimator, TransformerMixin):
    """
	Transformer that returns the number of sentences in the text.
	This feature was proven useful through grid-search.
	"""
    def __init__(self, used=True):
		# 'used' is boolean to decide if the feature is used or not.
        self.used = used

    def numsent_extractor(self, text):
        try:
            sentences = sent_tokenize(text)
            if self.used:
                return len(sentences)
            else:
                return 0
        except:
            return 0

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        X_tagged = pd.Series(X).apply(self.numsent_extractor)
        return pd.DataFrame(X_tagged)

def build_model(grid_search=1):
    """
    Model construction.

    Input :
    grid_search : if 1, the gridsearch is ran to find the best hyper-parameters. Warning : this is a long process.
    If 0, we take the model that was tuned.

    Output:
    cv : optimal model with fined-tuned hyper-parameters
    """
    if grid_search:

        scorer = make_scorer(generalized_fscore, greater_is_better = True)
        clf = MultiOutputClassifier(RandomForestClassifier())

        pipeline = 	Pipeline([

	        ('features', FeatureUnion([

	            ('nlp_pipeline', Pipeline([
	                ('vect', CountVectorizer(tokenizer=tokenize, max_df=0.75, max_features=1000, min_df=3, ngram_range=(1, 3))),
	                ('tfidf', TfidfTransformer())
	            ])),

	            ('dollars_extractor', DollarsExtractor()),
	            ('question_extractor', QuestionExtractor()),
	            ('numsent_extractor', NumSentExtractor())

	    	]))

	    ])

        parameters_loc = {
			'vect__ngram_range': [(1, 3)],
			'vect__max_df': (0.75, 1.0),
			'vect__max_features': (5000, 10000),
			'vect__min_df': (1,5),
			'clf__estimator__n_estimators': [50, 100, 200],
			'clf__estimator__min_samples_split': [2, 5]
		}

        cv = GridSearchCV(pipeline, param_grid=parameters_loc, scoring=scorer, n_jobs=-1, verbose=10)

    else:

        cv = Pipeline([

	        ('features', FeatureUnion([

	            ('nlp_pipeline', Pipeline([
	                ('vect', CountVectorizer(tokenizer=tokenize, max_df=0.75, max_features=1000, min_df=3, ngram_range=(1, 3))),
	                ('tfidf', TfidfTransformer())
	            ])),

	            ('dollars_extractor', DollarsExtractor()),
	            ('question_extractor', QuestionExtractor()),
	            ('numsent_extractor', NumSentExtractor())

	        ])),

	    	('clf', MultiOutputClassifier(RandomForestClassifier(min_samples_split=2, n_estimators=10)))

	    ])

    return cv

def evaluate_model(model, X_test, y_test, category_names):
    """
	Model evaluation.

	Input :
	model : model fit on data
	X_test : features of the test set.
	y_test : ground-truth on test set.
	category_names : names of categories

	Output:
	No output. The function computes and shows the model performance.
	"""

    y_pred = model.predict(X_test)

    acc = (y_pred == y_test).mean(axis=0)
    for ii, (name, acc_loc) in enumerate(zip(category_names, acc)):
        print('='*50)
        print('- '+name.upper()+':' )
        print('-'*50)
        print('Accuracy : {:.2f}'.format(acc_loc))
        print('-'*50)
        print(classification_report(y_test[:, ii], y_pred[:, ii]))

    pass

def save_model(model, model_filepath):
    """
    Model saving.
    """
    pickle.dump(model, open(model_filepath, 'wb'))

    pass

def main():
    """
	Main function.
	"""
    if len(sys.argv) == 3:

        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model(grid_search=0)

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


"""Callback handlers used in the app."""
from typing import Any, Coroutine, Dict, List, Optional
from uuid import UUID

from langchain.callbacks.base import AsyncCallbackHandler

from app.schemas import ChatResponse

class StreamingLLMCallbackHandler(AsyncCallbackHandler):
    """Callback handler for streaming LLM responses."""

    def __init__(self, websocket):
        self.websocket = websocket

    async def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
        resp = ChatResponse(sender="bot", message=token, type="stream")
        print(token)
        await self.websocket.send_json(resp.dict())

class QuestionGenCallbackHandler(AsyncCallbackHandler):
    """Callback handler for question generation."""

    def __init__(self, websocket):
        self.websocket = websocket

    async def on_llm_start(
        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
    ) -> None:
        """Run when LLM starts running."""
        resp = ChatResponse(
            sender="bot", message="Synthesizing question...", type="info"
        )
        await self.websocket.send_json(resp.dict())

    async def on_text(self, text: str, *, run_id: UUID, parent_run_id: UUID | None = None, **kwargs: Any) -> Coroutine[Any, Any, None]:
        print(text)
        return await super().on_text(text, run_id=run_id, parent_run_id=parent_run_id, **kwargs)

    async def on_llm_new_token(
        self,
        token: str,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> None:

        print(token)


from abc import ABC, abstractclassmethod
from app.models.model import DownloadModel
from concurrent.futures import ThreadPoolExecutor
import requests
import logging
import os
import tqdm
import sys

class ModelDownloadService(ABC):
    def __init__(
        self,
        download_models: list[DownloadModel],
        destination: str,
        chunk: int = 1024
    ) -> None:
        self.download_models = download_models
        self.destination = destination
        self.chunk = chunk

        if not os.path.isdir(self.destination):
            self._create_dir()

    def _get_model_dest_path(self, model: DownloadModel) -> str:
        if model.is_url_in_model_name:
            file_name = model.get_model_download_url().split('/')[-1]
            combined_path = os.path.join(self.destination, file_name)

        else:
            combined_path = os.path.join(self.destination, model.model_name)

        return combined_path

    def _save_model(self, model: DownloadModel) -> None:
        dest_path = self._get_model_dest_path(model)
        res = requests.get(model.download_model_url, stream=True)
        total_size_in_bytes = int(res.headers.get('content-length', 0))

        with tqdm.tqdm(total=int(total_size_in_bytes / self.chunk), unit='KB', unit_scale=True, file=sys.stdout, desc=model.model_name) as pbar:
            with open(dest_path, 'wb') as f:
                for data in res.iter_content(chunk_size=self.chunk):
                    f.write(data)
                    pbar.update(1)
                    f.flush()

    def _create_dir(self):
        try:
            os.mkdir(self.destination)

        except Exception as e:
            raise e

    @abstractclassmethod
    def run(self) -> None:
        pass

class ModelMultiThreadDownloadService(ModelDownloadService):
    def run(self) -> None:
        logging.basicConfig(level=logging.DEBUG, format='%(threadName)s: %(message)s')

        with ThreadPoolExecutor() as executor:
            for download_model in self.download_models:
                executor.submit(self._download_thread, download_model)

            executor.shutdown(wait=True)
        logging.debug('All tasks have been completed.')

    def _download_thread(self, model: DownloadModel):

        logging.debug(f'started {model.model_name} download.')
        self._save_model(model)
        logging.debug(f'Completed {model.model_name} download.')


from django.core.validators import MinValueValidator
from django.db import models
from django.db.models import UniqueConstraint

from users.models import User

class Ingredient(models.Model):
    """Модель для ингредиентов."""
    name = models.CharField(
        max_length=100,
        verbose_name='Название ингредиента',
        help_text='Укажите название ингредиента',
        null=False,
    )
    measurement_unit = models.CharField(
        max_length=100,
        verbose_name='Единица измерения',
        help_text='Укажите единицу измерения ингредиента',
        null=False,
    )

    class Meta:
        verbose_name = 'Ингредиент'
        verbose_name_plural = 'Ингредиенты'

    def __str__(self):
        return f'{self.name}, {self.measurement_unit}'

class Tag(models.Model):
    """Модель для тэгов блюд."""
    name = models.CharField(
        max_length=200,
        verbose_name='Название тега',
        null=False,
        unique=True
    )
    slug = models.SlugField(
        verbose_name='Ссылка',
        unique=True,
        help_text='Ссылка тега'
    )
    color = models.CharField(
        max_length=7,
        default='#ffffff',
        unique=True,
        verbose_name='Цвет тэга'
    )

    class Meta:
        verbose_name = 'Тэг'
        verbose_name_plural = 'Тэги'

    def __str__(self):
        return self.name

class Recipe(models.Model):
    """Модель для рецептов блюд."""
    author = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        related_name='recipes',
        verbose_name='Автор'
    )
    name = models.CharField(
        max_length=100,
        verbose_name='Название рецепта',
        help_text='Укажите название рецепта'
    )
    image = models.ImageField(
        upload_to='media/',
        verbose_name='Изображение блюда'
    )
    ingredients = models.ManyToManyField(
        Ingredient,
        related_name='ingredients',
        verbose_name='Ингредиенты'
    )
    text = models.TextField(
        verbose_name='Описание рецепта',
        help_text='Введите описание рецепта'
    )
    tags = models.ManyToManyField(
        Tag, related_name='tags',
        verbose_name='Тэг'
    )
    cooking_time = models.PositiveSmallIntegerField(
        default=1,
        validators=[MinValueValidator(1, 'Значение не может быть меньше 1')],
        verbose_name='Время готовки блюда в минутах',
    )

    class Meta:
        verbose_name = 'Рецепт'
        verbose_name_plural = 'Рецепты'

    def __str__(self):
        return self.name

class Favorite(models.Model):
    """Модель для избранного."""
    user = models.ForeignKey(
        User,
        related_name='favorites',
        on_delete=models.CASCADE
    )
    recipe = models.ForeignKey(
        Recipe,
        related_name='favorites',
        on_delete=models.CASCADE
    )
    added = models.DateTimeField(
        auto_now_add=True,
        verbose_name='Дата добавления'
    )

    class Meta:
        verbose_name = 'Избранное'
        verbose_name_plural = 'Избранное'
        UniqueConstraint(
            fields=['recipe', 'user'],
            name='favorite_unique'
        )

    def __str__(self):
        return f'У {self.user} в избранном: {self.recipe.name}'

class ShoppingList(models.Model):
    """Модель для списка покупок."""
    user = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        related_name='shopping_list',
        verbose_name='Пользователь'
    )
    recipe = models.ForeignKey(
        Recipe,
        on_delete=models.CASCADE,
        related_name='purchases',
        verbose_name='Покупка'
    )
    added = models.DateTimeField(
        auto_now_add=True,
        verbose_name='Дата добавления'
    )

    class Meta:
        verbose_name = 'Список покупок'
        verbose_name_plural = 'Список покупок'

    def __str__(self):
        return f'У {self.user} список покупок: {self.recipe}'

class AmountOfIngredient(models.Model):
    """Модель для количества ингредиентов."""
    ingredient = models.ForeignKey(
        Ingredient,
        on_delete=models.CASCADE,
        related_name='ingredients_in_recipe',
        verbose_name='Ингредиент'
    )
    recipe = models.ForeignKey(
        Recipe,
        on_delete=models.CASCADE,
        related_name='recipes_ingredients_list',
        verbose_name='Рецепт'
    )
    amount = models.PositiveSmallIntegerField(
        default=1,
        validators=[MinValueValidator(1)],
        verbose_name='Количество ингредиентов'
    )

    class Meta:
        verbose_name = 'Количество ингредиентов'
        verbose_name_plural = 'Количество ингредиентов'

    def __str__(self):
        return f'{self.ingredient} в {self.recipe}'


import requests

from bs4 import BeautifulSoup

def nouda( url , out ):

	r = requests.get( url )
	r.encoding = 'UTF-8'
	soup = BeautifulSoup( r.text )

	for teksti in soup.find_all( class_='field field-name-body field-type-text-with-summary field-label-hidden' ):
		for p in teksti.find_all( 'p' ):

			for string in p.stripped_strings:
	        		out.write(repr(string))

if __name__ == '__main__':

	nouda("http://www.uusisuomi.fi/kotimaa/79148-ex-ministeri-kummastelee-jotkut-pitavat-lahes-rikollisena", file('uusisuomi.txt', 'w'))


from django.db import models
from django.db.models import Avg, Count
from django.urls import reverse
from django.contrib.auth.models import User

class Recipe(models.Model):
    author = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
    name = models.CharField(max_length=200, null=True)
    CATEGORY_CHOICES = [
        ('Breakfast', 'Breakfast'),
        ('Lunch', 'Lunch'),
        ('Dinner', 'Dinner'),
        ('Snack', 'Snack'),
        ('Sweets', 'Sweets'),
    ]
    category = models.CharField(max_length=20, choices=CATEGORY_CHOICES, null=True)
    APPLIANCE_CHOICES = [
       ('Air Fryer', 'Air Fryer'),
       ('Microwave', 'Microwave'),
       ('Oven', 'Oven'),
       ('Pressure Cooker', 'Pressure Cooker'),
       ('Slow Cooker', 'Slow Cooker'),
       ('Stove', 'Stove'),
    ]
    appliance = models.CharField(max_length=20, choices=APPLIANCE_CHOICES, null=True)
    description = models.TextField(max_length=2000, null=True)
    time = models.IntegerField(null=True)
    servings = models.IntegerField(null=True)
    ingredients = models.TextField(max_length=3000, null=True)
    directions = models.TextField(max_length=3000, null=True)
    bookmarks = models.ManyToManyField(User, related_name='bookmarked_recipes', blank=True)

    def average_rating(self):
       return self.review_set.aggregate(Avg('rating'))['rating__avg']

    def total_reviews(self):
       return self.review_set.count()

    def __str__(self):
      return f'{self.name} ({self.id})'

    def get_absolute_url(self):
      return reverse('recipe_detail', kwargs={'pk': self.id})

class Review(models.Model):
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    content = models.TextField(max_length=2000)
    RATING_CHOICES = [
      (1, '1'),
      (2, '2'),
      (3, '3'),
      (4, '4'),
      (5, '5'),
    ]
    rating = models.IntegerField(choices=RATING_CHOICES, default=5)
    recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE)

    def __str__(self):
        return f'Review by {self.user.username}'

class Photo(models.Model):
    url = models.CharField(max_length=500)
    recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE)


# coding: utf-8

# In[17]:

try:
    import unicodecsv
except:
    get_ipython().system(u'pip install unicodecsv')
    import unicodecsv

# In[18]:

try:
    import numpy
except:
    get_ipython().system(u'pip install numpy')
    import numpy

# In[19]:

try:
    import nltk
except:
    get_ipython().system(u'pip install nltk')
    import nltk

# In[20]:

import pandas as pd
import numpy as np
import time
from sklearn.model_selection import cross_val_predict
from sklearn import linear_model
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.feature_extraction.text import TfidfVectorizer

# In[21]:

print("We here")
time1 = time.clock()
x_dataset = pd.read_csv("bitcoin_dataset.csv", delimiter=",", dtype=str, header=0, usecols=["Date"]).values
y_dataset = pd.read_csv("bitcoin_dataset.csv", delimiter=",", dtype=str, header=0, usecols=["dervative"]).values
date_headlines = pd.read_csv("abcnews-date-text.csv", delimiter=",", dtype=str, header=0, usecols=["publish_date"]).values
text_headlines = pd.read_csv("abcnews-date-text.csv", delimiter=",", dtype=str, header=0, usecols=["headline_text"]).values
time2 = time.clock()
print("Loading training set inputs took " + str(time2-time1) + " seconds")

x_dataset = x_dataset.tolist()
y_dataset = y_dataset.tolist()
date_headlines = date_headlines.tolist()
text_headlines = text_headlines.tolist()

# In[ ]:

for i in range(len(x_dataset)):
    string = str(x_dataset[i])
    string = string.replace("-","")
    x_dataset[i] = string[2:10]

# In[ ]:

for j in range(len(date_headlines)):
    string = str(date_headlines[j])
    string = string.replace("'","")
    string = string.replace("[","")
    string = string.replace("]","")
    date_headlines[j] = string
print(date_headlines[0])

# In[6]:

for k in range(len(text_headlines)):
    string = str(text_headlines[k])
    string = string.replace("'","")
    string = string.replace("[","")
    string = string.replace("]","")
    text_headlines[k] = string
print(text_headlines[0])

# In[7]:

j = 0
time1 = time.clock()
for f in x_dataset:
    stuff = ' '
    i = 0
    for m in date_headlines:
        if (m == f):
            stuff += text_headlines[i]
            stuff += ' '
        i = i + 1
    x_dataset[j] = stuff
    j = j + 1
time2 = time.clock()
print("Loading this took " + str(time2-time1) + " seconds")

# In[8]:

y_dataset[0] = 0
y_dataset[2605] = 0.5
for r in range(len(y_dataset)):
    string = str(y_dataset[r])
    string = string.replace("'","")
    string = string.replace("[","")
    string = string.replace("]","")
    y_dataset[r] = float(string)

# In[9]:

for r in range(len(y_dataset)):
    #Stability
    if(y_dataset[r] > -0.33 and y_dataset[r] < 0.53):
        y_dataset[r] = 2
        print(2)
    #Low
    if(y_dataset[r] < 2.50 and y_dataset[r] >= 0.53):
        y_dataset[r] = 3
        print(3)
    if(y_dataset[r] >= 2.50):
        y_dataset[r] = 4
        print(4)
    if(y_dataset[r] <= -1.91):
        y_dataset[r] = 0
        print(0)
    if(y_dataset[r] <= -0.33 and y_dataset[r] > -1.91):
        y_dataset[r] = 1
        print(1)

# In[10]:

def create_vectorizer():
    # Arguments here are tweaked for working with a particular data set.
    # All that's really needed is the input argument.
    return TfidfVectorizer(input='content', max_features=200,
                           max_df=0.05,
                           stop_words='english')

# In[11]:

x_dataset_try = x_dataset
vectorizer = create_vectorizer()
tfidf_result = vectorizer.fit_transform(x_dataset_try)

# In[12]:

print(y_dataset[7] == 4)
print(y_dataset[45])
print(len(y_dataset))

# In[13]:

n = 0
j = 0
for i in range(len(y_dataset)):
    if(y_dataset[j] == 3):
        n = n + 1
    j = j + 1
print(n)

# In[14]:

from sklearn.neural_network import MLPClassifier
MLP = MLPClassifier(random_state=1, max_iter=1000, learning_rate='adaptive')
X_train, X_test, y_train, y_test = train_test_split(tfidf_result, y_dataset, train_size=0.7, test_size=0.3)

# In[15]:

MLP.fit(X_train,y_train)
score = MLP.score(X_test,y_test) * 100
print("Test score: %.4f" % score)

# In[16]:

xtfidf2 = vectorizer.transform(np.array(['australia crisis senate concern problem high rates']))
MLP.predict(xtfidf2)


#!/usr/bin/env python

"""
test for TimeoutError

see: https://github.com/bluesky/ophyd/issues/776
"""

import datetime
import epics
import logging
import os
import sys
import time

for _nm in "epics epics.ca epics.pv".split():
    logger = logging.getLogger(_nm)
    logger.setLevel("DEBUG")

if len(sys.argv) == 1:
    CYCLES = 10
elif len(sys.argv) == 2:
    CYCLES = int(sys.argv[1])
DELAY_S = 1e-6
TEST_PV = "8idi:Reg200"
TIMEOUT = 1.0

pv = epics.PV(TEST_PV)
pv.wait_for_connection()

def move(signal, label, dest, delay_s):
    t0 = time.time()
    signal.put(dest, timeout=TIMEOUT, wait=True)
    dt = time.time() - t0
    info = signal.get_with_metadata()
    if info is None:
        raise TimeoutError("timeout")
    print(
        f"{datetime.datetime.now()} "
        f"MOVE: {label}:  {dest} {signal.get()}  {dt:.6f}"
        "\n# ----------------------------------"
    )
    time.sleep(delay_s)

i = 0
def ping_pong(signal, v1, v2, delay_s=1e-2):
    global i
    move(signal, f"ping {i+1}", v1, delay_s)
    move(signal, f"pong {i+1}", v2, delay_s)
    i += 1

if __name__ == "__main__":
    print(epics.__name__, epics.__version__)

    for cycle in range(CYCLES):
        ping_pong(pv, .1, -.1, delay_s=DELAY_S)

    print(epics.__name__, epics.__version__)


"""

Created by Ali Hajian

"""

import segyio
from time import time
import numpy as np

# class to create SGY reader objects
class SGY_Reader():
    # need to pass the file name for initialization
    def __init__(self, file_name):
        self.file_name = file_name
        with segyio.open(self.file_name) as src:
            self.ilines = list(src.ilines)
            self.xlines = list(src.xlines)
            self.trace_len = src.trace[0].size
            self.depths = range(self.trace_len)

    # function to select iline slices and load them from the sgy file into a numpy array
    def get_iline_slices(self, ilines):
        t = time()
        # make sure ilines are iterables
        try:
            n_ilines = len(ilines)
        except:
            ilines = [ilines]
            n_ilines = len(ilines)

        # make sure all ilines are valid
        for i in range(n_ilines):
            if ilines[i] not in self.ilines:
                print("iline %i is not valid" %ilines[i])
                return None

        # open and read the iline slices from the SGY file
        with segyio.open(self.file_name) as src:
            if n_ilines > 1:
                slice_list = np.empty(
                    (n_ilines, len(self.xlines), self.trace_len),
                    dtype=np.float32
                )
            for i in range(n_ilines):
                if n_ilines > 1:
                    slice_list[i] = src.iline[ilines[i]]
                else:
                    slice_list = src.iline[ilines[i]]

        print('--> %i iline slices from %s loaded in %.2f sec' %(n_ilines,
                                                                 self.file_name,
                                                                 time()-t))

        return slice_list

    # function to select xline slices and load them from the sgy file into a numpy array
    def get_xline_slices(self, xlines):
        t = time()
        # make sure xlines are iterables
        try:
            n_xlines = len(xlines)
        except:
            xlines = [xlines]
            n_xlines = len(xlines)

        # make sure all xlines are valid
        for i in range(n_xlines):
            if xlines[i] not in self.xlines:
                print("xline %i is not valid" %xlines[i])
                return None

        # open and read the xline slices from the SGY file
        with segyio.open(self.file_name) as src:
            if n_xlines > 1:
                slice_list = np.empty(
                    (n_xlines, len(self.ilines), self.trace_len),
                    dtype=np.float32
                )
            for i in range(n_xlines):
                if n_xlines > 1:
                    slice_list[i] = src.xline[xlines[i]]
                else:
                    slice_list = src.xline[xlines[i]]

        print('--> %i xline slices from %s loaded in %.2f sec' %(n_xlines,
                                                                 self.file_name,
                                                                 time()-t))

        return slice_list

    # function to select depth slices and load them from the sgy file into a numpy array
    def get_depth_slices(self, depths):
        t = time()
        # make sure depths are iterables
        try:
            n_depths = len(depths)
        except:
            depths = [depths]
            n_depths = len(depths)

        # make sure all depths are valid
        for i in range(n_depths):
            if depths[i] not in self.depths:
                print("depth %i is not valid" %depths[i])
                return None

        # open and read the depth slices from the SGY file
        with segyio.open(self.file_name) as src:
            if n_depths > 1:
                slice_list = np.empty(
                    (n_depths, len(self.ilines), len(self.xlines)),
                    dtype=np.float32
                )
            for i in range(n_depths):
                if n_depths > 1:
                    slice_list[i] = src.depth_slice[depths[i]]
                else:
                    slice_list = src.depth_slice[depths[i]]

        print('--> %i depth slices from %s loaded in %.2f sec' %(n_depths,
                                                                 self.file_name,
                                                                 time()-t))

        return slice_list

    # function to select traces and load them from the sgy file into a numpy array
    def get_traces(self, ilines, xlines):
        t = time()
        # make sure ilines and xlines are iterables
        try:
            n_ilines = len(ilines)
        except:
            ilines = [ilines]
            n_ilines = len(ilines)
        try:
            n_xlines = len(xlines)
        except:
            xlines = [xlines]
            n_xlines = len(xlines)

        # make sure ilines and xlines have the same size
        if n_ilines == n_xlines:
            n_traces = n_ilines
        else:
            print("Number of ilines and xlines need to be the same.")
            return None

        # make sure all ilines and xlines are valid
        for i in range(n_traces):
            if ilines[i] not in self.ilines:
                print("iline %i is not valid" %ilines[i])
                return None
            if xlines[i] not in self.xlines:
                print("xline %i is not valid" %xlines[i])
                return None

        # open and read the traces from the SGY file
        with segyio.open(self.file_name) as src:
            if n_traces > 1:
                trace_list = np.empty((n_traces, self.trace_len), dtype=np.float32)
            for i in range(n_traces):
                if n_traces > 1:
                    trace_list[i] = src.gather[ilines[i], xlines[i]]
                else:
                    trace_list = src.gather[ilines[i], xlines[i]][0]

        print('--> %i traces from %s loaded in %.2f sec' %(n_traces, self.file_name, time()-t))

        return trace_list

    # function to select slices or traces and load them from the sgy file into a numpy array
    def get_data(self, ilines=None, xlines=None, depths=None):
        if depths is not None:
            if ilines or xlines:
                print("Too many arguments. Can not use depths " \
                       "and ilines xlines at the same time.")
                return None
            else:
                # get depth slices
                data = self.get_depth_slices(depths)
        elif ilines is not None:
            if xlines is not None:
                # get the traces at the intersection of inlines and xlines
                data = self.get_traces(ilines, xlines)
            else:
                # get iline slices
                data = self.get_iline_slices(ilines)
        elif xlines is not None:
            # get xline slices
            data = self.get_xline_slices(xlines)
        else:
            print("Need to pass at least one of ilines or xlines or depths.")
            return None

        return data


import time
import pytest
import shutil
import os
from AppiumFramework.base.DriverClass import Driver

"""
This method will be called each time a testClass is executed
it sets the driver instance and quit it at the end of each
test case
"""
@pytest.fixture(scope="class")
def oneTimeSetUp(request):
    driver1 = Driver()
    driver = driver1.getDriverMethod()

    if request.cls is not None:
        request.cls.driver = driver

    yield driver
    driver.quit()

"""
This method will clean up our previous logs to have a clean report
and without previous runs information
"""
@pytest.fixture(scope="session")
def cleanLogs():
    logFile = open("Code2LEad.log", 'w')
    logFile.close()
    shutil.rmtree("/Users/rene.cortes/PycharmProjects/AppiumPython/AppiumFramework/reports/allureReports")
    # time.sleep(5)
    os.mkdir("/Users/rene.cortes/PycharmProjects/AppiumPython/AppiumFramework/reports/allureReports")


import datetime
import heapq
import sys

import networkx as nx
import numpy as np
import pandas as pd
import pickle
import json
import joblib

def init_real_graph():
    data = pd.read_csv('./data.csv')
    data = data.iloc[:, 1:]
    data = data.to_numpy()
    G = nx.Graph()
    for weight in data:
        G.add_edge(int(weight[0]), int(weight[1]), weight=weight[2])
    G.add_edge(0, 7388, weight=1.4108)

    return G

class PriorityQueue:
    def __init__(self):
        self.queue = []
        self.index = 0

    def __iter__(self):
        self.a = 1
        return self

    def __next__(self):
        x = self.a
        self.a += 1
        return x

    def empty(self):
        return len(self.queue) == 0  # if Q is empty

    def put(self, item, priority):
        heapq.heappush(self.queue, (priority, self.index, item))  # reorder x using priority
        self.index += 1

    def get(self):
        return heapq.heappop(self.queue)[-1]  # pop out element with smallest priority

class Label:
    def __init__(self, start, end):
        self.start = start
        self.end = end
        # 暂定为0
        self.timeList = [0]
        self.routes = [start]
        self.time = 0
        self.car_time = 0

def init_labels(query_num, graph_size):
    labels = []
    for i in range(query_num):
        start = np.random.randint(0, graph_size)
        end = np.random.randint(0, graph_size)
        if start == end:
            continue
        label = Label(start, end)
        labels.append(label)
    return labels

def get_flow(init_time, init_node, edge_dict, node):
    car_num = 0
    if (init_node, node) in edge_dict:
        for i in edge_dict[(init_node, node)]:
            if i[0] < init_time < i[1]:
                car_num += 1
    return car_num

def show_answer(res):
    total_cost = 0
    for label in res:
        total_cost += label.timeList[-1] - label.timeList[0]

def greedy_algorithms(labels, G, distances, alpha, edge_dict):
    global car_end, car_end
    pq = PriorityQueue()
    res = []
    for label in labels:
        pq.put(label, label.timeList[-1])

    while not pq.empty():
        label = pq.get()
        nodes = list(G.neighbors(label.routes[-1]))

        EGT_tmp = sys.maxsize
        init_time = label.timeList[-1]
        init_node = label.routes[-1]
        end_time = 0
        end_node = 0
        time_start = datetime.datetime.now()
        for node in nodes:
            # if distances.get(node) is not None and distances.get(init_node) is not None:
            if node not in label.routes and distances[node][label.end] < distances[init_node][label.end]:

                car_start_time = datetime.datetime.now()
                car_num = get_flow(init_time, init_node, edge_dict, node)
                car_end_time = datetime.datetime.now()

                label.car_time += (car_end_time - car_start_time).microseconds
                time_tmp = G[init_node][node]["weight"] * (1 + alpha * car_num)
                EGT = init_time - label.timeList[0] + time_tmp + distances[node][init_node]
                if EGT_tmp == sys.maxsize and EGT < EGT_tmp:
                    label.routes.append(node)
                    label.timeList.append(init_time + time_tmp)
                    end_node = node
                    end_time = init_time + time_tmp
                    EGT_tmp = EGT
                elif EGT_tmp != sys.maxsize and EGT < EGT_tmp:
                    label.routes[-1] = node
                    label.timeList[-1] = init_time + time_tmp
                    end_node = node
                    end_time = init_time + time_tmp
                    EGT_tmp = EGT
        time_end = datetime.datetime.now()

        # calculate time
        label.time += (time_end - time_start).microseconds
        if end_node != 0 and end_time != 0:
            if (init_node, end_node) in edge_dict:
                edge_dict[(init_node, end_node)].append([init_time, end_time])
            else:
                edge_dict[(init_node, end_node)] = [[init_time, end_time]]

        if label.routes[-1] == label.end:
            res.append(label)

        else:
            pq.put(label, label.timeList[-1])
    return res

def refining_algorithm(combination, G, distances, rate):
    pass

# def save_dict(filename, dic):
#     """save dict into json file"""
#     with open(filename, 'w') as json_file:
#         json.dump(dic, json_file, ensure_ascii=False)
#
#
# def load_dict(filename):
#     """load dict from json file"""
#     with open(filename, "r") as json_file:
#         dic = json.load(json_file)
#     return dic

def main():
    # 先设置场数，例如query_num等, graph_num=18262, 真实图中的点为18262
    query_num = 2000
    graph_size = 18262
    alpha = 0.02
    rate = 0.02

    G = init_real_graph()
    distances = dict(nx.all_pairs_dijkstra_path_length(G, weight="weight"))
    with open('distances.pkl', 'wb') as f:
        joblib.dump(distances, f, pickle.HIGHEST_PROTOCOL)

    start = datetime.datetime.now()
    with open('distances.pkl', 'rb') as f:
        distances = pickle.load(f)
        print(type(distances))

    end = datetime.datetime.now()
    print((end - start).seconds)
    # # save_dict('distances', distances)
    # distances = load_dict('distances')

    labels = init_labels(query_num=query_num, graph_size=graph_size)

    edge_dict = {}

    # 计算算法一时间和cost
    start_time = datetime.datetime.now()
    res = greedy_algorithms(labels, G, distances, alpha, edge_dict)
    end_time = datetime.datetime.now()
    print("time cost1" + str((end_time - start_time).seconds))
    print("cost1" + str(show_answer(res)))
    for i in range(len(res)):
        print(res[i].time)
        print("计算汽车时间" + str(res[i].car_time))

    # 计算算法二时间和cost
    # start_time = datetime.datetime.now()
    # combination2 = refining_algorithm(res, G, distances, rate)
    # end_time = datetime.datetime.now()
    # print("time cost2" + str((end_time - start_time).seconds))
    # print("cost2" + str(show_answer(res)))

if __name__ == '__main__':
    main()


import os
import re
import sys
# Add submodules to path
from datetime import datetime

from Common_classes.DBConnector import DBConnector, BLM, BeamInterval, Run, BeamMode
from multiprocessing import Pool
from source.Plotters.PlotCalc import PlotCalc

# from config import BLM_LIST_DIR, PICKLE_INTENSITY_INTERVALS_PATH, PLOTS_DIR_PATH
from source.Calculators.Integral.BeamModeSubIntervalsCalc import BeamModeSubIntervalsCalc
from source.Calculators.Integral.PostOffsetCorrectedIntegralCalc import PostOffsetCorrectedIntegralCalc
from source.Calculators.Integral.PreOffsetCorrectedIntegralCalc import PreOffsetCorrectedIntegralCalc
from source.Calculators.Integral.RawIntegralCalc import RawIntegralCalc
from source.Calculators.Offset.PostOffsetCalc import PostOffsetCalc
from source.Calculators.Offset.PreOffsetCalc import PreOffsetCalc
# from source.Plotters.BLMsPlotter import BLMsPlotter
from source.BLMProcess import BLMProcess
import logging
from arguments_parser import build_blm_dose_calc_parser
import pandas as pd
# from lhc_runs import lhc_runs, extract_runs

BLM_LIST_DIR = '/mnt/monitoring_analysis/data/blm_lists'

def save_to_excel(blms, fname='blms'):
    writer = pd.ExcelWriter(fname + '.xlsx')
    for blm in blms:
        blm.get_as_pandas_dataframe().to_excel(writer,blm.name)
    writer.save()

def save_to_excel_beam_modes(blms, fname, all_beam_modes):
    writer = pd.ExcelWriter(fname + '.xlsx')
    rslt = pd.concat([blm.get_beam_mode_doses_as_dataframe(all_beam_modes) for blm in blms], axis=0)
    rslt.to_excel(writer,'data')
    writer.save()

if __name__ == '__main__':
    dbc_test = DBConnector('pcen35754', db_name='lhc_raw', user='jenkins')
    dbc_test.connect_to_db()
    dbc_test.build_database()
    dbc_test.commit()
    runs = list(dbc_test.get_lhc_runs())
####################################################################################

    parser = build_blm_dose_calc_parser(runs)
    args = parser.parse_args()
    # print(args.start_date)

    dbc_test.close()
############################# Parsed command line arguments assignment ########
    # runs = Run.extract_runs(args)
    # start = runs[0][0]
    # end = runs[-1][1]
    requested_run = Run.extract_runs_from_command_line_argument(args, runs)
    blm_csv_list_filename = args.file_name
    number_of_simultaneous_processes = args.processes_num
    should_plot_total = args.pt
    should_plot_intensity_norm = args.pi
    should_plot_luminosity_norm = bool(args.pl)
    should_save_excel = args.save_to_excel
    luminosity = args.pl # 39.31 if start.year == 2016 else 37.83
    should_plot_cumsum = args.pcs
    logging_level = getattr(logging, args.logging_level)
    blm_raw_data_dir = args.raw_blm_path
    pickled_blm_dir = args.pickled_blm_path
    field = args.field_name
    IP_num = args.ir if args.ir else args.arc

    blm_filter_function_name = 'ir' if args.ir else None
    blm_filter_function_name = 'arc' if args.arc else None
    if not blm_filter_function_name:
        blm_filter_function_name = 'all'

    should_plot_heatmap = False
    left_offset = 700
    right_offset = 700

# ####################################################################################
    logging.basicConfig(level=logging_level)

    calculators = [PreOffsetCalc(), PreOffsetCorrectedIntegralCalc(),
                   RawIntegralCalc(),
                   PostOffsetCalc(), PostOffsetCorrectedIntegralCalc(),
                   BeamModeSubIntervalsCalc(),
                   # PlotCalc('plots')
                   ]

    dbc_test.connect_to_db()
    dbc_test.build_database()
    dbc_test.commit()

    if blm_csv_list_filename is not None:
        # blms_list = ['BLMED.04R8.B2C10_TDI.4R8.B2', ]
        blm_list_file_path = os.path.join(BLM_LIST_DIR, blm_csv_list_filename)
        blms_list = pd.read_csv(blm_csv_list_filename, header=None)[0].values
        filter_func = BLM.name.in_(blms_list)
    else:
        filter_func = True

    blms = list(dbc_test.session.query(BLM).filter(filter_func).all())
    # blms =list(filter(lambda  blm: '16L2' in blm.name or '17L2' in blm.name or '15L2' in blm.name, blms))
    print(len(blms))
    beam_modes = list(dbc_test.session.query(BeamMode))
    beam_intervals = list(filter(lambda beam_interval: beam_interval.start_time in requested_run, dbc_test.session.query(BeamInterval).all()))
    dbc_test.close()

    should_plot = should_plot_total or should_plot_cumsum or should_plot_intensity_norm or should_plot_luminosity_norm
    should_return_blm = should_plot or should_save_excel
    blm_process = BLMProcess(requested_run, field, calculators, should_return_blm, dbc_test, beam_intervals)
    # Reading and processing BLMs data
    # with Pool(processes=number_of_simultaneous_processes) as pool:
    #     blm_names_blm_intervals = {pseudoBLM.name: pseudoBLM.blm_intervals for pseudoBLM in pool.map(blm_process.run, blms[:]) if pseudoBLM is not None}

    blm_names_blm_intervals = {pseudoBLM.name: pseudoBLM.blm_intervals for pseudoBLM in list(map(blm_process.run, blms[:])) if pseudoBLM is not None}
    if blm_process.should_return_blm:

        blm_names = set(blm_names_blm_intervals.keys())
        for blm in blms:
            # print(blm.name, blm_names)
            if blm.name in blm_names:
                blm.blm_intervals_filtered = blm_names_blm_intervals[blm.name]
                # for i in blm.blm_intervals_filtered:
                #     print('{:%Y-%m-%d %H:%M}\t{:%Y-%m-%d %H:%M}\t{:.3e}\t{:.3e}'.format(i.start_time, i.end_time, i.integrated_dose, i.offset_pre))
            else:
                blm.blm_intervals_filtered = []
                # sys.exit()
        logging.info('Analysed BLM types: {}'.format(', '.join(set(blm.get_blm_type() for blm in blms))))

        if should_save_excel and blms:
            # save_to_excel_beam_modes(blms, 'BLMs_with_beam_modes_{}'.format(requested_run),beam_modes)
            save_to_excel_beam_modes(blms, 'BLMs_with_beam_modes_{:%Y%m%d}_{:%Y%m%d}'.format(requested_run.get_earliest_date(), requested_run.get_latest_date()),beam_modes)

        if should_plot:
            # Plotting
            pass
            # p = BLMsPlotter('.')
            # if should_plot_luminosity_norm:
            #     p.plot_luminosity_normalized_dose(blms, lambda blm: blm.get_pre_oc_dose(), luminosity)
            # if should_plot_intensity_norm:
            #     p.plot_intensity_normalized_dose(blms, lambda blm: blm.get_pre_oc_dose(), lambda blm: blm.get_oc_intensity_integral())
            # if should_plot_cumsum:
            #     p.plot_total_cumulated_dose(blms, lambda blm, start, end: blm.get_pre_oc_dose(start, end))
            # if should_plot_total:
            #     p.plot_total_dose(blms, lambda blm: blm.get_pre_oc_dose())
            # if should_plot_heatmap:
            #     p.heat_map_plot(blms)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Есть идея создать бота для девочек
То есть девочка переходит по ссылке, пишет боту, а он присылает ей рандомную открытку на 8 марта и какое-то поздравление
Открытки и поздравления сейчас в процессе

Девочка переходит по ссылке, дальше бот предлагает ей получить открытку или поздравление, девочка получает то,
что выбрала, потом можно еще выбрать. Можешь еще давать вопросы какие-то наводящие, например
"Привет, ты мальчик или девочка" , про курс или направление можно спросить

"""
import random
from vk_api.longpoll import VkEventType
import importlib
import botlib

HEADERS = {"user-agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.2"}

def _start():
    for event in longpoll.listen():
        if event.type == VkEventType.MESSAGE_NEW and event.to_me and event.text:
            sent_message = event.text
            user_id = event.user_id

            # Обычные сообщения. Ответ на них приоритетнее

            # ответ на начать
            if sent_message.lower() in botlib.HELLO_USER_MESSAGES:
                botlib.send(user_id, botlib.HELLO_ANSWER_MESSAGE % botlib.UserVK(user_id).get_name(), keyboard=botlib.json_main_kb)

            # ответ на просьбу получить открытку
            elif sent_message.lower() == botlib.CONST_JSON["keyboards"]["main"]["card"]["text"].lower():
                botlib.send(user_id, botlib.CONST_JSON["answers"]["yourCard"], attachment=random.choice(botlib.IMAGES_LIST))

            # ответ на просьбу получить поздравление
            elif sent_message.lower() == botlib.CONST_JSON["keyboards"]["main"]["greeting"]["text"].lower():
                botlib.send(user_id, random.choice(botlib.greetings_file))

            # ответ на просьбу получить описание пасхалки
            elif sent_message.lower() == botlib.CONST_JSON["keyboards"]["main"]["about_egg_1"]["text"].lower():
                botlib.send(user_id, botlib.about_egg_file)

            # обработка секретного сообщения, которое могут получить только девушки
            elif sent_message.lower() == botlib.CONST_JSON["answers"]["secretTriggerMessage"].lower():
                if botlib.UserVK(user_id).get_sex() == "женский":
                    botlib.send(user_id, botlib.CONST_JSON["answers"]["secretMessage"] % botlib.UserVK(user_id).get_name(), attachment=botlib.SECRET_IMAGE)
                    botlib.send(int(botlib.CONST_VK_ADMIN_ID),
                                f"🎁 [id{user_id}|{botlib.UserVK(user_id).get_name()} {botlib.UserVK(user_id).get_name('last')}] нашла пасхалку!")
                else:
                    botlib.send(user_id, botlib.HELLO_ERROR_MESSAGE)

            # обработка команды АВТОР
            elif sent_message.lower() == botlib.CONST_JSON["keyboards"]["main"]["author"]["text"].lower():
                botlib.send(user_id, botlib.CONST_JSON["answers"]["author"])

            # команды администратора
            elif str(user_id) in botlib.CONST_VK_ADMINLIST_COMMSEP and sent_message.lower()[0] == "/":
                command = sent_message.lower().split()
                botlib.send(user_id, f"🍕 Вы запустили операцию: "+command[0])

                if int(user_id) != int(botlib.CONST_VK_ADMIN_ID):
                    botlib.send(int(botlib.CONST_VK_ADMIN_ID), f"🍕 Администратор [id{user_id}|{botlib.UserVK(user_id).get_name()} {botlib.UserVK(user_id).get_name('last')}] запустил операцию: "+command[0])

                if command[0] == "/обновить":
                    """Команда перезагружает текстовый и графический контент бота.
                    Если были внесены изменения в ветке content, то надо выполнить эту команду.

                    Команда останавливает бота до окончания операции!"""
                    importlib.reload(botlib)
                    botlib.send(user_id, "🍕 Обновление бота завершено")
                    if int(user_id) != int(botlib.CONST_VK_ADMIN_ID):
                        botlib.send(int(botlib.CONST_VK_ADMIN_ID),
                                f"🍕 Администратор [id{user_id}|{botlib.UserVK(user_id).get_name()} {botlib.UserVK(user_id).get_name('last')}] завершил операцию: " +
                                command[0])
                    raise ValueError(command[0])

                elif command[0] == "/статус":
                    """Команда показывает отправителю свой статус"""
                    botlib.send(user_id, "🍕 Вы являетесь администратором чат-бота")

                elif command[0] == "/помощь":
                    """Команда показывает всю основную информацию о боте"""
                    botlib.send(user_id, botlib.HELP_ADMIN)

                elif command[0] == "/список_админов":
                    """Команда отправляет список администраторов

                    Команда останавливает бота до окончания операции!"""
                    admin_list = ""
                    for e in botlib.CONST_VK_ADMINLIST_COMMSEP:
                        admin_list += f"[id{e}|{botlib.UserVK(int(e)).get_name()} {botlib.UserVK(int(e)).get_name('last')}]\n"
                    botlib.send(user_id, "🍕 Список админов чат-бота:\n" + admin_list)

                elif command[0] == "/количество_диалогов":
                    """Команда отправляет список написавших пользователей. Не выполнять без особенной необходимости!"""
                    response = botlib.VK.method("messages.getConversations", {"access_token": botlib.CONST_JSON["tokenVK"]})
                    botlib.send(user_id, f"🍕 Количество диалогов с сообществом на данный момент: {response['count']}")

                else:
                    botlib.send(user_id, f"🍕 Команда " + command[0] + " не найдена. Попробуйте написать /помощь, чтобы получить список доступных команд.")
            else:
                botlib.send(user_id, botlib.HELLO_ERROR_MESSAGE)

while True:
    try:
        vk = botlib.VK
        longpoll = botlib.LONGPOLL
        upload = botlib.UPLOAD
        print('Бот запущен!')
        print(__doc__)
        _start()
    except Exception:
        pass


from collections import namedtuple
import json
import warnings

EntityField = namedtuple("EntityField", ["name", "isOptional"])

class WhatIfValidator():
    """
    Provides a simple way to validate that your entities will successfully
    upload.  Provides functions to validate the type, check if required
    attributes are missing, and check if superfluous attributes are included.

    :param dict type_defs:
        The list of type definitions to be validated against.  Should be
        in the form of an AtlasTypeDef composite wrapper.
    :param list(dict) existing_entities:
        The existing entities that should be validated against.
    """
    ASSET_ATTRIBUTES = ["name", "description", "owner"]
    REFERENCABLE_ATTRIBUTES = ["qualifiedName"]

    ATLAS_MODEL = {
        "ASSET": ["name", "description", "owner"],
        "REFERENCABLE": ["qualifiedName"],
        "PROCESS": (["inputs", "outputs"] +
                    ASSET_ATTRIBUTES + REFERENCABLE_ATTRIBUTES),
        "DATASET": ASSET_ATTRIBUTES + REFERENCABLE_ATTRIBUTES,
        "INFRASTRUCTURE": ASSET_ATTRIBUTES + REFERENCABLE_ATTRIBUTES
    }

    def __init__(self, type_defs={}, existing_entities=[]):
        if len(type_defs) == 0 and len(existing_entities) == 0:
            warnings.warn(
                "WARNING: Provided type_defs and existing_entities are empty.  All validations will pass.")

        self.classification_defs = type_defs.get("classificationDefs", [])
        self.entity_defs = type_defs.get("entityDefs", [])
        # Create a dict of all entities by name, then find the name of all
        # attributes and whether they are optional
        entity_fields = {}
        for e in self.entity_defs:
            entity_fields[e["name"]] = []
            for attr in e.get("attributeDefs", []):
                ef = EntityField(attr.get("name"), attr.get("isOptional"))
                entity_fields[e["name"]].append(ef)

        # Adding Qualified Name to the set of valid fields as it doesn't
        # show up in the entity type def
        self.entity_valid_fields = {k: set(
            [field.name for field in v] + ["qualifiedName"])
            for k, v in entity_fields.items()}
        # Adding Qualified Name to the set of required entity
        # fields as it doesn't show up in the entity type def
        self.entity_required_fields = {
            k: set([field.name for field in v if not field.isOptional] + [
                "qualifiedName"]) for k, v in entity_fields.items()
        }

        self.existing_entities = set(
            [e.get("attributes", {}).get("qualifiedName") for
             e in existing_entities])

        self.enum_defs = type_defs.get("enumDefs", [])
        self.relationship_defs = type_defs.get("relationshipDefs", [])
        self.struct_defs = type_defs.get("structDefs", [])

    def entity_type_exists(self, entity):
        """
        Validate that the entity's type is an actual entity definition.

        :param dict entity:
        :return: Whether the entity matches the list of known entity types.
        :rtype: bool
        """
        current_type = entity["typeName"]
        if current_type in self.entity_valid_fields:
            return True
        else:
            return False

    def entity_missing_attributes(self, entity):
        """
        Check if the entity is missing required attributes.

        :param dict entity:
        :param dict type_def:
        :return: Whether the entity matches the list of known entity types.
        :rtype: bool
        """

        current_attributes = set(entity.get("attributes", {}).keys())
        required_attributes = set(
            self.entity_required_fields[entity["typeName"]])
        missing_attributes = required_attributes.difference(current_attributes)
        if len(missing_attributes) > 0:
            return missing_attributes
        else:
            return False

    def entity_has_invalid_attributes(self, entity):
        """
        Check if the entity is using attributes that are not defined
        on the type.

        :param dict entity:
        :return: Whether the entity matches the list of known entity types.
        :rtype: bool
        """
        current_attributes = set(entity.get("attributes", {}).keys())
        valid_attributes = set(self.entity_valid_fields[entity["typeName"]])
        # Append inherited attributes:
        _entity_type = entity["typeName"]
        # Assuming only one entity matches and only one super type
        super_type = [e["superTypes"]
                      for e in self.entity_defs
                      if e["name"] == _entity_type][0][0].upper()
        if super_type in self.ATLAS_MODEL:
            valid_attributes = valid_attributes.union(
                self.ATLAS_MODEL[super_type])
        invalid_attributes = current_attributes.difference(valid_attributes)

        if len(invalid_attributes) > 0:
            return invalid_attributes
        else:
            return False

    def entity_would_overwrite(self, entity):
        """
        Based on the qualified name attributes, does the provided
        entity exist in the entities provided to the What If Validator?

        :param dict entity:
        :return: Whether the entity matches an existing entity.
        :rtype: bool
        """
        current_qualifiedName = entity.get(
            "attributes", {}).get("qualifiedName")
        if current_qualifiedName is None:
            raise KeyError("Entity has no qualifiedName:\n{}".format(
                json.dumps(entity, indent=1)))

        if current_qualifiedName in self.existing_entities:
            return True
        else:
            return False

    def validate_entities(self, entities):
        """
        Provide a report of invalid entities.  Includes TypeDoesNotExist,
        UsingInvalidAttributes, and MissingRequiredAttributes.

        :param list(dict) entities: A list of entities to validate.
        :return: A dictionary containing counts values for the above values.
        :rtype: dict
        """
        report = {"TypeDoesNotExist": [], "UsingInvalidAttributes": {},
                  "MissingRequiredAttributes": {}}

        for entity in entities:
            if not self.entity_type_exists(entity):
                report["TypeDoesNotExist"].append(entity["guid"])
                # If it's an invalid type, we have to skip over the rest
                # of this
                continue

            using_invalid = self.entity_has_invalid_attributes(entity)
            is_missing = self.entity_missing_attributes(entity)
            cur_guid = entity["guid"]
            if using_invalid:
                report["UsingInvalidAttributes"][cur_guid] = using_invalid

            if is_missing:
                report["MissingRequiredAttributes"][cur_guid] = is_missing

        output = {
            "counts": {k: len(v) for k, v in report.items()},
            "values": report
        }
        output.update({"total": sum(output["counts"].values())})

        return output


#!/usr/bin/env python
import cv2
from openni2_device_init import visionsensor
import numpy as np
from PIL import Image
from io import BytesIO
import time
import zlib

class VideoCamera(object):
    def __init__(self,x = 640, y = 480, fps = 30, rgb_mirror = False,
            depth_mirror = False, rgb = True, depth = True):
        self.device = visionsensor(x = x, y = y, fpd = fps, rgb_mirror = rgb_mirror,
                depth_mirror = depth_mirror, rgb = rgb, depth = depth)
        self.rgb = rgb
        self.depth = depth
        time.sleep(1)

    def __del__(self):
        self.device.stop()

    def get_frame(self, rgb_compress = 75):
        jpeg = ''
        depth = ''
        if self.rgb:
            rgb = self.device.getRgb()
            #img = Image.fromarray(rgb)
            #fpath =BytesIO()
            #img.save(fpath, quality = 75, format = "JPEG")
            #fpath.seek(0)
            encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), rgb_compress]
            ret, jpeg = cv2.imencode('.jpg', rgb, encode_param)
            jpeg = jpeg.tobytes()
        if self.depth:
            depth = zlib.compress(self.device.getDepth2Int8())
        return jpeg, depth


"""
Jon William Taylor
Date: November 1 2021
Assignment: Visualization Output and Line Graph

About the program:
This program will read data from a file, draw a line graph from it, and display the output

Input: Will take survey data from a file and process it

Output: Will draw a line graph from the survey data
"""
print(__doc__)

### import statements here

### define sub functions here

### the main function

from matplotlib import pyplot as plt

import csv
import numpy as np

#get two lists of random numbers
from random import randint

x = [] #places the data for the x axis
y1 = [] #places the data for the first y axis
y2 = [] #places the data for the second y axis
def line_graph():
    with open(r'Demographic_Stats_By_Zip_Code.csv') as csv_file: #opens the file
        plot_values = csv.reader(csv_file, delimiter = ",")
        header = next(plot_values) #starts plotting the values
        counter = 0 #starts the process of reading the values
        for row in plot_values:
            counter += 1 #counts each row for data
            #if counter > 20:
                #break
            x.append(int(row[0]))
            y1.append(int(row[2])) #total female participants
            y2.append(int(row[3])) #total male participants

    the_header = str(header).strip('[')
    the_header = str(the_header).strip(']')
    headers = the_header.split(',') #splits the data

    for header in headers:
        h = header.lstrip("'") #splits the data
        h = h.rstrip("'") #splits the data
        #print(h)
    #title_plots = title_plots.strip("'")
    print(headers[0])
    print(headers[2])
    print(headers[3])

    xValues = np.arange(len(x)) #starts arranging the values on the graph
    plt.plot(xValues, y1, linestyle='--', marker='o', c='red', linewidth=2, markersize=5, mec='black', mfc='blue', label="Female") #determines the characteristics of first y axis
    plt.plot(xValues, y2, linestyle=':', marker='v', c='green', linewidth=2, markersize=5, mec='blue', mfc='yellow', label='Male') #determines the characteristics of second y axis
    plt.legend() #places the legend on the graph
    plt.xlabel(headers[0].strip().strip("'"))
    plt.ylabel(headers[1].strip().strip("'"))
    plt.xticks(np.arange(len(x)), x) #arranges the data into a graph
    #plt.set_xtickslabels(x, rotation=90)
    plt.title("Female and Male Survey Participants") #names the top of the graph
    plt.tight_layout()
    plt.savefig(r"C:\Users\Desktop\OutputGraph.png") #outputs the graph into a png file at file location
    plt.show() #shows the graph as a png file

def main():
    line_graph()

if __name__ == '__main__':
    main()

ZipCode/PA06 Visualization Output and Line Graph.py


"""
# My first app
Here's our first attempt at using data to create a table:
"""

import streamlit as st
import pandas as pd
from prediction import prediction

DATASET_FILE = "./cleaned_dataset.csv"
dataset_df = pd.read_csv(DATASET_FILE)

n = st.slider('Number of topics to show', 1, 15)

text = st.text_input('Review', 'Type here...')

if st.button('Predict'):
    topics = prediction(text, n)

    if type(topics) is str:
        st.write(topics)
    else:
        for i, topic in enumerate(topics):
            st.write(f"{i + 1} : {topic}")