from pymongo import MongoClient
from wa_api import WA_API

# Collection Names
AC = "archers"
CC = "competitions"
QC = "qualifications"
QAC = "qualifications_arrows"

class MongoManage:
    def __init__(self, host='localhost', port=27017, rs=None):
        if rs:
            self.client = MongoClient(host=host, port=port, replicaset=rs, readPreference='primaryPreferred')
        else:
            self.client = MongoClient(host=host, port=port)
        self.db = None

    def set_database(self, db_name='wa'):
        self.db = self.client[db_name]

    def insert(self, collection, obj):
        """
        Insert for the collections with no dependencies with other collections
        OR where dependencies has already been resolved
        """
        try:
            result = self.db[collection].insert_one(obj)
            return result.inserted_id
        except:
            print("{0} collection: failed to insert object: {1}".format(collection, obj))
            return -1

    def insert_qualification(self, qualification):
        competition_id = self.db[CC].find_one({'wa_id': qualification['competition_id']})['_id']
        qualification['competition_id'] = competition_id

        wa_archer_ids = [aid for aid in qualification['archer_ids']]
        archer_ids = []
        for wa_ai in wa_archer_ids:
            try:
                aid = self.db[AC].find_one({'wa_id': wa_ai})['_id']
            except TypeError:
                # if no such archer is found in MongoDB, find him via API and add him
                print("Archer with {0} World Archer ID was not found in the DB, inserting it...")
                wa = WA_API()
                archer = wa.db__get_single_archer(wa_ai)
                aid = self.insert(AC, archer)
                print("...inserting of archer is done, _id: {0}".format(aid))
            archer_ids.append(aid)
        qualification['archer_ids'] = archer_ids

        try:
            result = self.db[QC].insert_one(qualification)
            return result.inserted_id
        except:
            print("Qualifications collection: failed to insert qualification: {0}".format(qualification))
            return -1

    def get_qualifications(self, individual_team=None):
        """
        :param individual_team:
            1 - return only individual qualification results
            2 - return only team qualification results
            (others) - return both
        """
        if individual_team == 1:
            qualifications = self.db[QC].find({"is_team": 0})
        elif individual_team == 2:
            qualifications = self.db[QC].find({"is_team": 1})
        else:
            qualifications = self.db[QC].find()

        # populate the Competitions and Archers Collections
        qualifications = list(qualifications)
        for i in range(0, len(qualifications)):
            qualifications[i]['competition_id'] = self.db[CC].find_one({
                "_id": qualifications[i]['competition_id']
            })['wa_id']

            qualifications[i]['archer_ids'] = [self.db[AC].find_one({
                "_id": aid
            })['wa_id'] for aid in qualifications[i]['archer_ids']]

        return qualifications

    def get_arrows_within_competition(self, competition_wa_id):
        competition_id = self.db[CC].find_one({"wa_id": competition_wa_id})['_id']

        qualifications = self.db[QC].find({"competition_id": competition_id, "is_team": 0})
        qualification_ids = [q['_id'] for q in qualifications]

        qualification_arrows = self.db[QAC].find({"qualification_id": {"$in": qualification_ids}})
        qualification_arrows = [qa['arrows'] for qa in qualification_arrows]

        arrows = []
        for arrows_list in qualification_arrows:
            arrows.extend(arrows_list)
        return arrows

    def get_competitions(self):
        competitions = self.db[CC].find()
        return list(competitions)

    def get_individual_qualification_scores_within_competition(self, competition_wa_id):
        competition_id = self.db[CC].find_one({"wa_id": competition_wa_id})['_id']
        qualifications = self.db[QC].find({"competition_id": competition_id, "is_team": 0})
        return list(qualifications)

    def get_maximum_individual_qualification_score(self):
        male = self.db[QC].find({'is_team': 0, 'category': 'RM'}).sort([('score', -1)]).limit(1)[0]
        female = self.db[QC].find({'is_team': 0, 'category': 'RW'}).sort([('score', -1)]).limit(1)[0]

        # populate the Competitions And Archers Collections
        male['competition_id'] = self.db[CC].find_one({'_id': male['competition_id']})
        female['competition_id'] = self.db[CC].find_one({'_id': female['competition_id']})
        male['archer_ids'] = self.db[AC].find_one({'_id': male['archer_ids'][0]})
        female['archer_ids'] = self.db[AC].find_one({'_id': female['archer_ids'][0]})
        return {
            "male": male,
            "female": female,
        }

    def get_archer_results(self, archer_wa_id):
        archer = self.db[AC].find_one({"wa_id": archer_wa_id})
        qualifications = self.db[QC].find({"archer_ids": archer['_id']})
        qualifications = list(qualifications)

        # populate the Competitions Collection
        for i in range(0, len(qualifications)):
            qualifications[i]['competition_id'] = self.db[CC].find_one({'_id': qualifications[i]['competition_id']})
        return {
            "archer": archer,
            "qualifications": qualifications,
        }

    def get_country_results(self, NOC):
        qualifications = self.db[QC].aggregate([
            {
                "$unwind": "$archer_ids",
            },
            {
                "$lookup":
                {
                    "from": AC,
                    "localField": "archer_ids",
                    "foreignField": "_id",
                    "as": "archers",
                },
            },
            {
                "$match": {"{0}.NOC".format(AC): NOC},
            },
        ])

        # The Mongo Request above does return a little broken results
        # So that's why we have to adjust and combine them a bit
        qualifications = list(qualifications)
        unique_qualifications = list({q['_id']: q for q in qualifications}.values())
        for q in qualifications:
            for i in range(0, len(unique_qualifications)):
                if q['_id'] == unique_qualifications[i]['_id']:
                    for archer in unique_qualifications[i]['archers']:
                        if archer['wa_id'] == q['archers'][0]['wa_id']:
                            break
                    else:
                        unique_qualifications[i]['archers'].append(q['archers'][0])

        # For each of unique qualifications,
        # populate the Competitions Collection
        # and delete the unnecessary "archer_ids" field
        for i in range(0, len(unique_qualifications)):
            unique_qualifications[i]['competition_id'] = self.db[CC].find_one({"_id": unique_qualifications[i]['competition_id']})
            try:
                del unique_qualifications[i]['archer_ids']
            except KeyError:
                pass

        return unique_qualifications


import dash_core_components as dcc
import dash_html_components as html
from dash_devices.dependencies import Input, Output, State
import numpy as np
import pandas as pd
import sys
import re
import num2words

from app import app
from apps import user_mode

sys.path.insert(1, 'C:\\Users\\Antoine\\CloudStation\\EPFL\\Master 4\\Master project\\masterProject')
from UIDatasheet import UIDatasheet
import ui_finder

ui_path = 'C:\\Users\\Antoine\\CloudStation\\EPFL\\Master 4\\Master project\\Dataset\\VINS Dataset\\ui_list.json'
ui_df = pd.read_json(ui_path)

current_page = 0
max_page = 0
filtered_ui = []
image_selected = ''
images = ['', '', '', '', '', '', '', '']

description_layout = html.Div([
    html.Div([
        html.H3('Descriptions'),
        html.P(id='content-description', className='ow'),
        html.Button('Copy description', id='button-copy-description', n_clicks=0),
    ], style={'margin': '15px'}),
    html.Div([
        html.H3('AI search'),
    ], style={'margin': '15px'}),
    html.Div([
        html.Div([
            dcc.Textarea(id='content-text-value', value='', cols=70, placeholder='Text value'),
        ], style={'flex-grow': '1'}),
        html.Div([
            dcc.Input(id='input-topk', type='number', value=16, min=1, placeholder='Top-k value'),
        ], style={'flex-grow': '1'}),
        html.Div([
            html.Button('Run', id='button-run-ai', n_clicks=0),
        ], style={'flex-grow': '1'}),
    ], style={'margin': '15px', 'display': 'flex'}),
    html.Div([
        html.P(id='content-info-retrieved'),
    ], style={'margin': '15px'}),
    html.Div([
        html.Div([
            html.H3('Label'),
            dcc.Dropdown(
                id='dropdown-label',
                options=[
                    {'label': 'Bare', 'value': 'bare'},
                    {'label': 'Shop', 'value': 'shop'},
                    {'label': 'Form', 'value': 'form'},
                    {'label': 'Gallery', 'value': 'gallery'},
                    {'label': 'List', 'value': 'list'},
                    {'label': 'Login', 'value': 'login'},
                    {'label': 'Map', 'value': 'map'},
                    {'label': 'Menu', 'value': 'menu'},
                    {'label': 'Modal', 'value': 'modal'},
                    {'label': 'News', 'value': 'news'},
                    {'label': 'Profile', 'value': 'profile'},
                    {'label': 'Search', 'value': 'search'},
                    {'label': 'Settings', 'value': 'settings'},
                    {'label': 'Terms', 'value': 'terms'},
                    {'label': 'Tutorial', 'value': 'tutorial'},
                    {'label': 'Other', 'value': 'other'},
                ],
            ),
        ], style={'flex-grow': '1', 'margin': '15px'}),
        html.Div([
            html.H3('Number of buttons'),
            dcc.Input(id='input-buttons', type='number', min=0),
        ], style={'flex-grow': '1', 'margin': '15px'}),
        html.Div([
            html.H3('Number of input fields'),
            dcc.Input(id='input-input-fields', type='number', min=0),
        ], style={'flex-grow': '1', 'margin': '15px'}),
        html.Div([
            html.H3('Page indicator'),
            dcc.Dropdown(
                id='dropdown-page-indicator',
                options=[
                    {'label': 'Yes', 'value': 'yes'},
                    {'label': 'No', 'value': 'no'},
                ],
            ),
        ], style={'flex-grow': '1', 'margin': '15px'}),
        html.Div([
            html.H3('Map'),
            dcc.Dropdown(
                id='dropdown-map',
                options=[
                    {'label': 'Yes', 'value': 'yes'},
                    {'label': 'No', 'value': 'no'},
                ],
            ),
        ], style={'flex-grow': '1', 'margin': '15px'}),
        html.Div([
            html.H3('Text filter'),
            dcc.Textarea(id='content-text-filter', value='', rows=7),
        ], style={'flex-grow': '2', 'margin': '15px'}),
    ], style={'display': 'flex'}),
    html.Div([
        html.Button('Clear filters', id='button-clear-filters', n_clicks=0, style={'margin': '15px'}),
        html.Button('Search', id='button-search', n_clicks=0, style={'margin': '15px'}),
    ], style={'margin-bottom': '10px',
              'textAlign': 'center',
              'margin': 'auto'}),
    html.Div([
        html.Progress(id='progress-search', value='0', max=100, style={'width': '30%'}),
        html.P(id='content-search'),
    ], style={'margin-bottom': '10px',
              'textAlign': 'center',
              'margin': 'auto'}),
    html.Div([
        html.Div([
            html.Div([
                html.H3('Image selections'),
                html.Div([
                    html.Img(src=app.get_asset_url('background.png'), id='img_select0', style={'width': '20%'}),
                    html.Button('Select 1st image', id='button-select0', n_clicks=0, style={'height': '10%'}),
                    html.P('None', id='content-select0'),
                    html.Button('Clear', id='button-clear0', n_clicks=0, style={'height': '10%'}),
                ], style={'display': 'flex', 'margin': '15px'}),
                html.Div([
                    html.Img(src=app.get_asset_url('background.png'), id='img_select1', style={'width': '20%'}),
                    html.Button('Select 2nd image', id='button-select1', n_clicks=0, style={'height': '10%'}),
                    html.P('None', id='content-select1'),
                    html.Button('Clear', id='button-clear1', n_clicks=0, style={'height': '10%'}),
                ], style={'display': 'flex', 'margin': '15px'}),
                html.Div([
                    html.Img(src=app.get_asset_url('background.png'), id='img_select2', style={'width': '20%'}),
                    html.Button('Select 3rd image', id='button-select2', n_clicks=0, style={'height': '10%'}),
                    html.P('None', id='content-select2'),
                    html.Button('Clear', id='button-clear2', n_clicks=0, style={'height': '10%'}),
                ], style={'display': 'flex', 'margin': '15px'}),
                html.Div([
                    html.Img(src=app.get_asset_url('background.png'), id='img_select3', style={'width': '20%'}),
                    html.Button('Select 4th image', id='button-select3', n_clicks=0, style={'height': '10%'}),
                    html.P('None', id='content-select3'),
                    html.Button('Clear', id='button-clear3', n_clicks=0, style={'height': '10%'}),
                ], style={'display': 'flex', 'margin': '15px'}),
                # html.Div([
                #     html.Button('Select 5th image', id='button-select4', n_clicks=0),
                #     html.P('None', id='content-select4')
                # ], style={'display': 'flex', 'margin': '15px'}),
                # html.Div([
                #     html.Button('Select 6th image', id='button-select5', n_clicks=0),
                #     html.P('None', id='content-select5')
                # ], style={'display': 'flex', 'margin': '15px'}),
                # html.Div([
                #     html.Button('Select 7th image', id='button-select6', n_clicks=0),
                #     html.P('None', id='content-select6')
                # ], style={'display': 'flex', 'margin': '15px'}),
                # html.Div([
                #     html.Button('Select 8th image', id='button-select7', n_clicks=0),
                #     html.P('None', id='content-select7')
                # ], style={'display': 'flex', 'margin': '15px'}),
                html.Button('Send images', id='button-send-images', n_clicks=0),
            ], style={'margin': '15px'})
        ], style={'float': 'left', 'width': '20%'}),
        html.Div([
            html.Div(
                [
                    html.Div([
                        html.Img(src=app.get_asset_url('background.png'), id='img_god0', className='imageUI',
                                 n_clicks_timestamp=-1),
                    ], style={'flex-grow': '1', 'margin': '15px'}),
                    html.Div([
                        html.Img(src=app.get_asset_url('background.png'), id='img_god1', className='imageUI',
                                 n_clicks_timestamp=-1),
                    ], style={'flex-grow': '1', 'margin': '15px'}),
                    html.Div([
                        html.Img(src=app.get_asset_url('background.png'), id='img_god2', className='imageUI',
                                 n_clicks_timestamp=-1),
                    ], style={'flex-grow': '1', 'margin': '15px'}),
                    html.Div([
                        html.Img(src=app.get_asset_url('background.png'), id='img_god3', className='imageUI',
                                 n_clicks_timestamp=-1),
                    ], style={'flex-grow': '1', 'margin': '15px'}),
                ], style={'display': 'flex'}
            ),
            html.Div(
                [
                    html.Div([
                        html.Img(src=app.get_asset_url('background.png'), id='img_god4', className='imageUI',
                                 n_clicks_timestamp=-1),
                    ], style={'flex-grow': '1', 'margin': '15px'}),
                    html.Div([
                        html.Img(src=app.get_asset_url('background.png'), id='img_god5', className='imageUI',
                                 n_clicks_timestamp=-1),
                    ], style={'flex-grow': '1', 'margin': '15px'}),
                    html.Div([
                        html.Img(src=app.get_asset_url('background.png'), id='img_god6', className='imageUI',
                                 n_clicks_timestamp=-1),
                    ], style={'flex-grow': '1', 'margin': '15px'}),
                    html.Div([
                        html.Img(src=app.get_asset_url('background.png'), id='img_god7', className='imageUI',
                                 n_clicks_timestamp=-1),
                    ], style={'flex-grow': '1', 'margin': '15px'}),
                ], style={'display': 'flex'}
            ),
            html.Div(
                [
                    html.Div([
                        html.Button('Previous page', id='button-previous', n_clicks=0),
                    ], style={'flex-grow': '1', 'textAlign': 'right', 'margin': '15px'}),
                    html.Div([
                        html.Button('Next page', id='button-next-page', n_clicks=0),
                    ], style={'flex-grow': '1', 'textAlign': 'left', 'margin': '15px'}),
                ], style={'display': 'flex'}
            ),
            html.Div(
                [
                    html.P('Page ... out of ...', id='content-page-number'),
                ], style={'textAlign': 'center', 'margin': 'auto'}
            ),
        ], style={'float': 'right', 'width': '80%'}),
    ]),
])

commands_layout = html.Div([
    html.Div([
        html.H3('Selected Images'),
        html.P(id='content-image', className='ow'),
    ], style={'margin': '15px'}),
    html.Div([
        html.H3('Commands'),
        html.P(id='content-command', className='ow'),
    ], style={'margin': '15px'}),
    html.Div(
        [
            html.Button('Send error message', id='button-send-error', n_clicks=0, style={'margin': '15px'}),
            html.Button('Clear error message', id='button-clear-error', n_clicks=0, style={'margin': '15px'}),
        ],
        style={'margin-bottom': '10px',
               'textAlign': 'center',
               'width': '220px',
               'margin': 'auto'}
    ),
])

layout = html.Div([
    dcc.Tabs(id='tabs-god-mode', value='tab-description', children=[
        dcc.Tab(label='User\'s descriptions', value='tab-description', children=description_layout),
        dcc.Tab(label='User\'s commands', value='tab-commands', children=commands_layout),
    ])
])

@app.callback(None,
              [Input('button-send-error', 'n_clicks')])
def send_error(n_clicks):
    if n_clicks:
        app.push_mods({'content-error': {'children': 'Error: the requested command is too complex for the system, please modify your request'}})

@app.callback(None,
              [Input('button-clear-error', 'n_clicks')])
def send_error(n_clicks):
    if n_clicks:
        app.push_mods({'content-error': {'children': ''}})

@app.callback(None,
              [Input('button-select0', 'n_clicks')])
def select_image(n_clicks):
    if n_clicks:
        global image_selected
        if image_selected:
            images[0] = image_selected
            app.push_mods({'content-select0': {'children': image_selected}})
            app.push_mods({'img_select0': {'src': app.get_asset_url('wireframes/' + images[0])}})

@app.callback(None,
              [Input('button-select1', 'n_clicks')])
def select_image(n_clicks):
    if n_clicks:
        global image_selected
        if image_selected:
            images[1] = image_selected
            app.push_mods({'content-select1': {'children': image_selected}})
            app.push_mods({'img_select1': {'src': app.get_asset_url('wireframes/' + images[1])}})

@app.callback(None,
              [Input('button-select2', 'n_clicks')])
def select_image(n_clicks):
    if n_clicks:
        global image_selected
        if image_selected:
            images[2] = image_selected
            app.push_mods({'content-select2': {'children': image_selected}})
            app.push_mods({'img_select2': {'src': app.get_asset_url('wireframes/' + images[2])}})

@app.callback(None,
              [Input('button-select3', 'n_clicks')])
def select_image(n_clicks):
    if n_clicks:
        global image_selected
        if image_selected:
            images[3] = image_selected
            app.push_mods({'content-select3': {'children': image_selected}})
            app.push_mods({'img_select3': {'src': app.get_asset_url('wireframes/' + images[3])}})

@app.callback(None,
              [Input('button-clear0', 'n_clicks')])
def select_image(n_clicks):
    if n_clicks:
        global images
        images[0] = ''
        app.push_mods({'content-select0': {'children': 'None'}})
        app.push_mods({'img_select0': {'src': app.get_asset_url('background.png')}})

@app.callback(None,
              [Input('button-clear1', 'n_clicks')])
def select_image(n_clicks):
    if n_clicks:
        global images
        images[1] = ''
        app.push_mods({'content-select1': {'children': 'None'}})
        app.push_mods({'img_select1': {'src': app.get_asset_url('background.png')}})

@app.callback(None,
              [Input('button-clear2', 'n_clicks')])
def select_image(n_clicks):
    if n_clicks:
        global images
        images[2] = ''
        app.push_mods({'content-select2': {'children': 'None'}})
        app.push_mods({'img_select2': {'src': app.get_asset_url('background.png')}})

@app.callback(None,
              [Input('button-clear3', 'n_clicks')])
def select_image(n_clicks):
    if n_clicks:
        global images
        images[3] = ''
        app.push_mods({'content-select3': {'children': 'None'}})
        app.push_mods({'img_select3': {'src': app.get_asset_url('background.png')}})

@app.callback(None,
              [Input('button-copy-description', 'n_clicks')])
def copy_description(n_clicks):
    if n_clicks:
        if user_mode.data.descriptions:
            description = user_mode.data.descriptions[-1]
            app.push_mods({'content-text-value': {'value': description}})

@app.callback(None,
              [Input('img_god0', 'n_clicks_timestamp'),
               Input('img_god1', 'n_clicks_timestamp'),
               Input('img_god2', 'n_clicks_timestamp'),
               Input('img_god3', 'n_clicks_timestamp'),
               Input('img_god4', 'n_clicks_timestamp'),
               Input('img_god5', 'n_clicks_timestamp'),
               Input('img_god6', 'n_clicks_timestamp'),
               Input('img_god7', 'n_clicks_timestamp')])
def select_image(n_clicks_timestamp0, n_clicks_timestamp1, n_clicks_timestamp2, n_clicks_timestamp3,
                 n_clicks_timestamp4, n_clicks_timestamp5, n_clicks_timestamp6, n_clicks_timestamp7):
    if filtered_ui:
        global current_page
        global image_selected
        list_timestamps = [n_clicks_timestamp0, n_clicks_timestamp1, n_clicks_timestamp2, n_clicks_timestamp3,
                           n_clicks_timestamp4, n_clicks_timestamp5, n_clicks_timestamp6, n_clicks_timestamp7]
        max_idx = np.argmax(list_timestamps)
        if sum(list_timestamps) != -1 * len(list_timestamps):
            if max_idx + current_page * 8 < len(filtered_ui):
                image_selected = filtered_ui[max_idx + current_page * 8]
                app.push_mods({'img_god0': {'className': 'imageUI'}})
                app.push_mods({'img_god1': {'className': 'imageUI'}})
                app.push_mods({'img_god2': {'className': 'imageUI'}})
                app.push_mods({'img_god3': {'className': 'imageUI'}})
                app.push_mods({'img_god4': {'className': 'imageUI'}})
                app.push_mods({'img_god5': {'className': 'imageUI'}})
                app.push_mods({'img_god6': {'className': 'imageUI'}})
                app.push_mods({'img_god7': {'className': 'imageUI'}})

                app.push_mods({'img_god{}'.format(max_idx): {'className': 'imageUIselected'}})

@app.callback(None,
              [Input('button-send-images', 'n_clicks')])
def send_images(n_clicks):
    if n_clicks:
        if images[0]:
            app.push_mods({'img0': {'hidden': False}})
            app.push_mods({'img0': {'src': app.get_asset_url('wireframes/' + images[0])}})
        else:
            app.push_mods({'img0': {'hidden': True}})
        if images[1]:
            app.push_mods({'img1': {'hidden': False}})
            app.push_mods({'img1': {'src': app.get_asset_url('wireframes/' + images[1])}})
        else:
            app.push_mods({'img1': {'hidden': True}})
        if images[2]:
            app.push_mods({'img2': {'hidden': False}})
            app.push_mods({'img2': {'src': app.get_asset_url('wireframes/' + images[2])}})
        else:
            app.push_mods({'img2': {'hidden': True}})
        if images[3]:
            app.push_mods({'img3': {'hidden': False}})
            app.push_mods({'img3': {'src': app.get_asset_url('wireframes/' + images[3])}})
        else:
            app.push_mods({'img3': {'hidden': True}})
        # app.push_mods({'img4': {'src': app.get_asset_url('wireframes/' + images[4])}})
        # app.push_mods({'img5': {'src': app.get_asset_url('wireframes/' + images[5])}})
        # app.push_mods({'img6': {'src': app.get_asset_url('wireframes/' + images[6])}})
        # app.push_mods({'img7': {'src': app.get_asset_url('wireframes/' + images[7])}})
        user_mode.data.images_selected = images
        user_mode.data.image_sent = True

#
# @app.callback(None,
#               [Input('tabs-god-mode', 'value'), ])
# def render_content(tab):
#     if tab == 'tab-description':
#         app.push_mods({'tabs-layout': {'children': [description_layout]}})
#         description_content = ''
#         if user_mode.data.descriptions:
#             nb_desc = len(user_mode.data.descriptions)
#             for i in range(nb_desc):
#                 description_content += '{}: {}
'.format(i + 1, user_mode.data.descriptions[i])
#         app.push_mods({'content-description': {'children': DangerouslySetInnerHTML(description_content)}})
#     elif tab == 'tab-commands':
#         app.push_mods({'tabs-layout': {'children': [commands_layout]}})
#         image_content = ''
#         if user_mode.data.images:
#             nb_desc = len(user_mode.data.images)
#             for i in range(nb_desc):
#                 image_content += '{}: {}
'.format(i + 1, user_mode.data.images[i].replace('.jpg', ''))
#         app.push_mods({'content-image': {'children': DangerouslySetInnerHTML(image_content)}})
#
#         command_content = ''
#         if user_mode.data.commands:
#             nb_desc = len(user_mode.data.commands)
#             for i in range(nb_desc):
#                 command_content += '{}: {}
'. format(i+1, user_mode.data.commands[i])
#         app.push_mods({'content-command': {'children': DangerouslySetInnerHTML(command_content)}})

@app.callback(None,
              [Input('button-previous', 'n_clicks')])
def control_previous(n_clicks_previous):
    global current_page
    global max_page
    global filtered_ui
    if max_page:
        if n_clicks_previous:
            if current_page > 0:
                current_page -= 1
                app.push_mods(
                    {'img_god0': {'src': app.get_asset_url('wireframes/' + filtered_ui[0 + current_page * 8])}})
                app.push_mods(
                    {'img_god1': {'src': app.get_asset_url('wireframes/' + filtered_ui[1 + current_page * 8])}})
                app.push_mods(
                    {'img_god2': {'src': app.get_asset_url('wireframes/' + filtered_ui[2 + current_page * 8])}})
                app.push_mods(
                    {'img_god3': {'src': app.get_asset_url('wireframes/' + filtered_ui[3 + current_page * 8])}})
                app.push_mods(
                    {'img_god4': {'src': app.get_asset_url('wireframes/' + filtered_ui[4 + current_page * 8])}})
                app.push_mods(
                    {'img_god5': {'src': app.get_asset_url('wireframes/' + filtered_ui[5 + current_page * 8])}})
                app.push_mods(
                    {'img_god6': {'src': app.get_asset_url('wireframes/' + filtered_ui[6 + current_page * 8])}})
                app.push_mods(
                    {'img_god7': {'src': app.get_asset_url('wireframes/' + filtered_ui[7 + current_page * 8])}})
                app.push_mods(
                    {'content-page-number': {'children': 'Page {} out of {}'.format(current_page + 1, max_page)}})

@app.callback(None,
              [Input('button-clear-filters', 'n_clicks')])
def clear_filters(n_clicks):
    if n_clicks:
        app.push_mods({'dropdown-label': {'value': ''}})
        app.push_mods({'input-buttons': {'value': ''}})
        app.push_mods({'input-input-fields': {'value': ''}})
        app.push_mods({'dropdown-page-indicator': {'value': ''}})
        app.push_mods({'dropdown-map': {'value': ''}})
        app.push_mods({'content-text-filter': {'value': ''}})
        global max_page
        max_page = 0
        global filtered_ui
        filtered_ui = []

@app.callback(None,
              [Input('button-next-page', 'n_clicks')])
def control_next(n_clicks_next):
    global current_page
    global max_page
    global filtered_ui
    if max_page:
        if n_clicks_next:
            if current_page < max_page - 1:
                current_page += 1
                app.push_mods({'img_god0': {'src': app.get_asset_url('background.png')}})
                app.push_mods({'img_god1': {'src': app.get_asset_url('background.png')}})
                app.push_mods({'img_god2': {'src': app.get_asset_url('background.png')}})
                app.push_mods({'img_god3': {'src': app.get_asset_url('background.png')}})
                app.push_mods({'img_god4': {'src': app.get_asset_url('background.png')}})
                app.push_mods({'img_god5': {'src': app.get_asset_url('background.png')}})
                app.push_mods({'img_god6': {'src': app.get_asset_url('background.png')}})
                app.push_mods({'img_god7': {'src': app.get_asset_url('background.png')}})
                if len(filtered_ui) > 0 + current_page * 8:
                    app.push_mods(
                        {'img_god0': {'src': app.get_asset_url('wireframes/' + filtered_ui[0 + current_page * 8])}})
                if len(filtered_ui) > 1 + current_page * 8:
                    app.push_mods(
                        {'img_god1': {'src': app.get_asset_url('wireframes/' + filtered_ui[1 + current_page * 8])}})
                if len(filtered_ui) > 2 + current_page * 8:
                    app.push_mods(
                        {'img_god2': {'src': app.get_asset_url('wireframes/' + filtered_ui[2 + current_page * 8])}})
                if len(filtered_ui) > 3 + current_page * 8:
                    app.push_mods(
                        {'img_god3': {'src': app.get_asset_url('wireframes/' + filtered_ui[3 + current_page * 8])}})
                if len(filtered_ui) > 4 + current_page * 8:
                    app.push_mods(
                        {'img_god4': {'src': app.get_asset_url('wireframes/' + filtered_ui[4 + current_page * 8])}})
                if len(filtered_ui) > 5 + current_page * 8:
                    app.push_mods(
                        {'img_god5': {'src': app.get_asset_url('wireframes/' + filtered_ui[5 + current_page * 8])}})
                if len(filtered_ui) > 6 + current_page * 8:
                    app.push_mods(
                        {'img_god6': {'src': app.get_asset_url('wireframes/' + filtered_ui[6 + current_page * 8])}})
                if len(filtered_ui) > 7 + current_page * 8:
                    app.push_mods(
                        {'img_god7': {'src': app.get_asset_url('wireframes/' + filtered_ui[7 + current_page * 8])}})
                app.push_mods(
                    {'content-page-number': {'children': 'Page {} out of {}'.format(current_page + 1, max_page)}})

@app.callback(None,
              [Input('button-search', 'n_clicks')],
              [State('dropdown-label', 'value'),
               State('input-buttons', 'value'),
               State('input-input-fields', 'value'),
               State('dropdown-page-indicator', 'value'),
               State('dropdown-map', 'value'),
               State('content-text-filter', 'value')])
def filter_ui(n_clicks, label, nb_buttons, nb_input, page, map_, text_filter):
    if n_clicks:
        index_list = list(range(len(ui_df)))
        global filtered_ui
        filtered_ui = []
        if label:
            drop = []
            count = 0
            app.push_mods({'content-search': {'children': 'Label filtering...'}})
            for index in index_list:
                progress = int((count / len(index_list)) * 100)
                count += 1
                if progress % 10 == 0:
                    app.push_mods({'progress-search': {'value': str(progress)}})
                if ui_df.iloc[index].label != label:
                    drop.append(index)
            index_list = list(set(index_list) - set(drop))
            app.push_mods({'progress-search': {'value': '100'}})
            app.push_mods({'content-search': {'children': ''}})

        if nb_buttons:
            drop = []
            count = 0
            app.push_mods({'content-search': {'children': 'Button filtering...'}})
            for index in index_list:
                progress = int((count / len(index_list)) * 100)
                count += 1
                if progress % 10 == 0:
                    app.push_mods({'progress-search': {'value': str(progress)}})
                components = pd.DataFrame.from_dict(ui_df.iloc[index].components)
                nb_buttons_ui = components.type.str.count('TextButton').sum()
                if nb_buttons_ui != nb_buttons:
                    drop.append(index)
            index_list = list(set(index_list) - set(drop))
            app.push_mods({'progress-search': {'value': '100'}})
            app.push_mods({'content-search': {'children': ''}})

        if nb_input:
            drop = []
            count = 0
            app.push_mods({'content-search': {'children': 'Input fields filtering...'}})
            for index in index_list:
                progress = int((count / len(index_list)) * 100)
                count += 1
                if progress % 10 == 0:
                    app.push_mods({'progress-search': {'value': str(progress)}})
                components = pd.DataFrame.from_dict(ui_df.iloc[index].components)
                nb_input_ui = components.type.str.count('EditText').sum()
                if nb_input_ui != nb_input:
                    drop.append(index)
            index_list = list(set(index_list) - set(drop))
            app.push_mods({'progress-search': {'value': '100'}})
            app.push_mods({'content-search': {'children': ''}})

        if page:
            drop = []
            count = 0
            app.push_mods({'content-search': {'children': 'Page indicator filtering...'}})
            for index in index_list:
                progress = int((count / len(index_list)) * 100)
                count += 1
                if progress % 10 == 0:
                    app.push_mods({'progress-search': {'value': str(progress)}})
                components = pd.DataFrame.from_dict(ui_df.iloc[index].components)
                nb_page_ui = components.type.str.count('PageIndicator').sum()
                if page == 'yes':
                    if nb_page_ui == 0:
                        drop.append(index)
                if page == 'no':
                    if nb_page_ui > 0:
                        drop.append(index)
            index_list = list(set(index_list) - set(drop))
            app.push_mods({'progress-search': {'value': '100'}})
            app.push_mods({'content-search': {'children': ''}})

        if map_:
            drop = []
            count = 0
            app.push_mods({'content-search': {'children': 'Map filtering...'}})
            for index in index_list:
                progress = int((count / len(index_list)) * 100)
                count += 1
                if progress % 10 == 0:
                    app.push_mods({'progress-search': {'value': str(progress)}})
                components = pd.DataFrame.from_dict(ui_df.iloc[index].components)
                nb_map_ui = components.type.str.count('Map').sum()
                if map_ == 'yes':
                    if nb_map_ui == 0:
                        drop.append(index)
                if map_ == 'no':
                    if nb_map_ui > 0:
                        drop.append(index)
            index_list = list(set(index_list) - set(drop))
            app.push_mods({'progress-search': {'value': '100'}})
            app.push_mods({'content-search': {'children': ''}})

        if text_filter:
            drop = []
            count = 0
            app.push_mods({'content-search': {'children': 'Text filtering...'}})
            text_filter_words = text_filter.lower().split()
            for index in index_list:
                progress = int((count / len(index_list)) * 100)
                count += 1
                if progress % 10 == 0:
                    app.push_mods({'progress-search': {'value': str(progress)}})
                components = pd.DataFrame.from_dict(ui_df.iloc[index].components)
                text_ui = ' '.join(components.text.tolist()).lower()
                if not all(text in text_ui for text in text_filter_words):
                    drop.append(index)
            index_list = list(set(index_list) - set(drop))
            app.push_mods({'progress-search': {'value': '100'}})
            app.push_mods({'content-search': {'children': ''}})

        for index in index_list:
            filtered_ui.append(ui_df.iloc[index, 0] + '.jpg')

        app.push_mods({'img_god0': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god1': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god2': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god3': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god4': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god5': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god6': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god7': {'src': app.get_asset_url('background.png')}})

        if len(filtered_ui) == 0:
            app.push_mods({'content-search': {'children': 'No results found for the given filters.'}})
        if len(filtered_ui) > 0:
            app.push_mods({'img_god0': {'src': app.get_asset_url('wireframes/' + filtered_ui[0])}})
        if len(filtered_ui) > 1:
            app.push_mods({'img_god1': {'src': app.get_asset_url('wireframes/' + filtered_ui[1])}})
        if len(filtered_ui) > 2:
            app.push_mods({'img_god2': {'src': app.get_asset_url('wireframes/' + filtered_ui[2])}})
        if len(filtered_ui) > 3:
            app.push_mods({'img_god3': {'src': app.get_asset_url('wireframes/' + filtered_ui[3])}})
        if len(filtered_ui) > 4:
            app.push_mods({'img_god4': {'src': app.get_asset_url('wireframes/' + filtered_ui[4])}})
        if len(filtered_ui) > 5:
            app.push_mods({'img_god5': {'src': app.get_asset_url('wireframes/' + filtered_ui[5])}})
        if len(filtered_ui) > 6:
            app.push_mods({'img_god6': {'src': app.get_asset_url('wireframes/' + filtered_ui[6])}})
        if len(filtered_ui) > 7:
            app.push_mods({'img_god7': {'src': app.get_asset_url('wireframes/' + filtered_ui[7])}})

        page_size = 8
        global max_page
        global current_page
        current_page = 0
        max_page = int((len(filtered_ui) + page_size - 1) / page_size)
        if max_page == 0:
            max_page = 1
        app.push_mods({'content-page-number': {'children': 'Page {} out of {}'.format(current_page + 1, max_page)}})

@app.callback(None,
              [Input('button-run-ai', 'n_clicks')],
              [State('content-text-value', 'value'),
               State('input-topk', 'value')])
def run_ai(n_clicks, description, k):
    if n_clicks and description and k:
        app.push_mods({'content-info-retrieved': {'children': 'Running...'}})
        description = re.sub(r"(\d+)", lambda x: num2words.num2words(int(x.group(0))), description)
        ui_datasheet = UIDatasheet()
        ui_datasheet.description = description
        ui_finder.get_label(ui_datasheet)
        ui_finder.get_components(ui_datasheet)
        info = ui_finder.print_info(ui_datasheet)
        app.push_mods({'content-info-retrieved': {'children': info}})

        wf_list = ui_finder.search_wf(ui_datasheet, k)

        global filtered_ui
        filtered_ui = []
        for wf in wf_list:
            filtered_ui.append(wf + '.jpg')

        app.push_mods({'img_god0': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god1': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god2': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god3': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god4': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god5': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god6': {'src': app.get_asset_url('background.png')}})
        app.push_mods({'img_god7': {'src': app.get_asset_url('background.png')}})

        if len(filtered_ui) == 0:
            app.push_mods({'content-search': {'children': 'No results found for the given filters.'}})
        if len(filtered_ui) > 0:
            app.push_mods({'img_god0': {'src': app.get_asset_url('wireframes/' + filtered_ui[0])}})
        if len(filtered_ui) > 1:
            app.push_mods({'img_god1': {'src': app.get_asset_url('wireframes/' + filtered_ui[1])}})
        if len(filtered_ui) > 2:
            app.push_mods({'img_god2': {'src': app.get_asset_url('wireframes/' + filtered_ui[2])}})
        if len(filtered_ui) > 3:
            app.push_mods({'img_god3': {'src': app.get_asset_url('wireframes/' + filtered_ui[3])}})
        if len(filtered_ui) > 4:
            app.push_mods({'img_god4': {'src': app.get_asset_url('wireframes/' + filtered_ui[4])}})
        if len(filtered_ui) > 5:
            app.push_mods({'img_god5': {'src': app.get_asset_url('wireframes/' + filtered_ui[5])}})
        if len(filtered_ui) > 6:
            app.push_mods({'img_god6': {'src': app.get_asset_url('wireframes/' + filtered_ui[6])}})
        if len(filtered_ui) > 7:
            app.push_mods({'img_god7': {'src': app.get_asset_url('wireframes/' + filtered_ui[7])}})

        page_size = 8
        global max_page
        global current_page
        current_page = 0
        max_page = int((len(filtered_ui) + page_size - 1) / page_size)
        if max_page == 0:
            max_page = 1
        app.push_mods({'content-page-number': {'children': 'Page {} out of {}'.format(current_page + 1, max_page)}})


import pandas as pd
import streamlit as st
import numpy as np

from common import session_manager
ssm = session_manager.st_session()
# ssm.write_session_info()

st.title("表を描画する")

# データフレームを元に表を表示する
df = pd.DataFrame({
  'first column': [1, 2, 3, 4],
  'second column': [10, 20, 30, 40]
})

st.write("write関数")
st.write(df)

st.write("table関数")
st.table(df)

st.write("dataframe関数、writeとほとんど同じ？")
st.dataframe(df)

#
df2 = np.random.randn(10, 20)
st.dataframe(df2)

#
dataframe = pd.DataFrame(
    np.random.randn(10, 20),
    columns=('col %d' % i for i in range(20)))  # col名を設定

# highlight_max（最大値にハイライトする）、axis=0（インデックス（列？）の対して評価する、１とすると行になる）
# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.io.formats.style.Styler.highlight_max.html#pandas.io.formats.style.Styler.highlight_max
st.dataframe(dataframe.style.highlight_max(axis=1,color="red"))

#
st.table(dataframe.style.highlight_max(axis=1,color="red"))


import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from deconstruct_lc import read_config
from deconstruct_lc import tools_fasta
from deconstruct_lc import tools_lc
from deconstruct_lc.scores.norm_score import NormScore

class RemovePfam(object):
    def __init__(self):
        config = read_config.read_config()
        self.data_dp = os.path.join(config['fps']['data_dp'])
        self.puncta = os.path.join(self.data_dp, 'experiment', 'puncta_uni.fasta')
        self.nopuncta = os.path.join(self.data_dp, 'experiment', 'nopuncta_uni.fasta')
        self.pfam_puncta = os.path.join(self.data_dp, 'experiment', 'puncta_pfam.tsv')
        self.pfam_nopuncta = os.path.join(self.data_dp, 'experiment', 'nopuncta_pfam.tsv')
        self.k = 6
        self.lce = 1.6
        self.lca = 'SGEQAPDTNKR'
        self.lc_m = 0.06744064704548541
        self.lc_b = 16.5

    def run_percent_pfam(self):
        puncta_perc = os.path.join(self.data_dp, 'experiment', 'puncta_percent_pfam.tsv')
        self.percent_pfam(self.puncta, self.pfam_puncta, puncta_perc)
        nopuncta_perc = os.path.join(self.data_dp, 'experiment', 'nopuncta_percent_pfam.tsv')
        self.percent_pfam(self.nopuncta, self.pfam_nopuncta, nopuncta_perc)

    def percent_pfam(self, fasta_fp, pfam_fp, fpo):
        df = pd.read_csv(pfam_fp, sep='\t')
        pids, seqs = tools_fasta.fasta_to_id_seq(fasta_fp)
        frac_pfam = []
        for id, seq in zip(pids, seqs):
            ndf = df[df['uniprot_acc'] == id]
            ndf = ndf.sort_values(by='seq_start')
            segmented = self.segment_seq(seq, ndf)
            len_seg = 0
            for seg in segmented:
                len_seg += len(seg)
            frac_pfam.append(float(len(seq) - len_seg)/float(len(seq)))
        ns = NormScore()
        scores = ns.lc_norm_score(seqs)
        df_out = pd.DataFrame({'Uniprot ID': pids, 'LC Score': scores,
                               'Pfam Fraction': frac_pfam}, columns=['Uniprot ID', 'LC Score', 'Pfam Fraction'])
        df_out = df_out.sort_values(by='LC Score', ascending=False)
        df_out.to_csv(fpo, sep='\t')
        print(np.mean(frac_pfam))

    def run_with_pfam(self):
        puncta_out = os.path.join(self.data_dp, 'experiment', 'puncta_nopfam.tsv')
        self.with_pfam(self.puncta, self.pfam_puncta, puncta_out)
        nopuncta_out = os.path.join(self.data_dp, 'experiment', 'nopuncta_nopfam.tsv')
        self.with_pfam(self.nopuncta, self.pfam_nopuncta, nopuncta_out)

    def with_pfam(self, fasta_fp, pfam_fp, fpo):
        """
        How many proteins in the set have pfam domains?
        What is the fraction occupied by pfam domains?"""
        df = pd.read_csv(pfam_fp, sep='\t')
        pfam_ids = list(set(df['uniprot_acc']))
        pids, seqs = tools_fasta.fasta_to_id_seq(fasta_fp)
        print(len(pids))
        nopfam_ids = list(set(pids) - set(pfam_ids))
        nopfam_seqs = []
        for pid, seq in zip(pids, seqs):
            if pid in nopfam_ids:
                nopfam_seqs.append(seq)
        ns = NormScore()
        scores = ns.lc_norm_score(nopfam_seqs)
        df_out = pd.DataFrame({'UniProt ID': nopfam_ids, 'LC Score': scores}, columns=['UniProt ID', 'LC Score'])
        df_out = df_out.sort_values(by='LC Score', ascending=False)
        df_out.to_csv(fpo, sep='\t')

    def fetch_score(self, df, pids):
        scores = []
        for pid in pids:
            df = df[df['Protein ID'] == pid]
            scores.append(list(df['LC Score'])[0])
        return scores

    def score_in_pfam(self):
        ids, seqs = tools_fasta.fasta_to_id_seq(self.nopuncta)
        df = pd.read_csv(self.pfam_nopuncta, sep='\t', index_col=0)
        below = 0
        above = 0
        norm_scores = []
        fl_norm_scores = []
        for id, seq in zip(ids, seqs):
            ndf = df[df['uniprot_acc'] == id]
            ndf = ndf.sort_values(by='seq_start')
            segmented = self.pfam_segments(seq, ndf)
            total = 0
            for item in segmented:
                total += len(item)
            if total >= 100:
                above += 1
                fl_score, fl_length = self.get_segment_scores([seq])
                fl_norm = self.norm_function([fl_score], [fl_length])
                raw_score, length = self.get_segment_scores(segmented)
                norm_score = self.norm_function([raw_score], [length])
                norm_scores.append(norm_score[0])
                fl_norm_scores.append(fl_norm[0])
            else:
                below += 1
        print(above)
        print(below)
        print(np.mean(norm_scores))
        print(np.mean(fl_norm_scores))
        print(np.median(norm_scores))
        print(np.median(fl_norm_scores))
        plt.hist(fl_norm_scores, alpha=0.5, bins=20, range=(-100, 200), label='Full length scores')
        plt.hist(norm_scores, alpha=0.5, bins=20, range=(-100, 200), label='Inside Pfam scores')
        plt.legend()
        plt.show()

    def run(self):
        ids, seqs = tools_fasta.fasta_to_id_seq(self.puncta)
        df = pd.read_csv(self.pfam_puncta, sep='\t', index_col=0)
        new_seqs = []
        below = 0
        above = 0
        norm_scores = []
        fl_norm_scores = []
        for id, seq in zip(ids, seqs):
            ndf = df[df['uniprot_acc'] == id]
            ndf = ndf.sort_values(by='seq_start')
            segmented = self.segment_seq(seq, ndf)
            total = 0
            for item in segmented:
                total += len(item)
            if total >= 100:
                above += 1
                fl_score, fl_length = self.get_segment_scores([seq])
                fl_norm = self.norm_function([fl_score], [fl_length])
                raw_score, length = self.get_segment_scores(segmented)
                norm_score = self.norm_function([raw_score], [length])
                norm_scores.append(norm_score[0])
                fl_norm_scores.append(fl_norm[0])
            else:
                below += 1
        print(above)
        print(below)
        print(np.mean(norm_scores))
        print(np.mean(fl_norm_scores))
        print(np.median(norm_scores))
        print(np.median(fl_norm_scores))
        plt.hist(fl_norm_scores, alpha=0.5, bins=20, range=(-100, 200), label='Full length scores')
        plt.hist(norm_scores, alpha=0.5, bins=20, range=(-100, 200), label='Outside Pfam scores')
        plt.legend()
        plt.show()

    def pfam_segments(self, seq, df):
        new_seq = []
        for i, row in df.iterrows():
            new_seq.append(seq[row['seq_start']: row['seq_end']+1])
        return new_seq

    def segment_seq(self, seq, df):
        """Given intervals, pull out the domain, and segment around it"""
        start = 0
        new_seq = []
        for i, row in df.iterrows():
            new_seq.append(seq[start:row['seq_start']])
            start = row['seq_end'] + 1
        new_seq.append(seq[start:])
        return new_seq

    def pfam_in_common(self):
        df = pd.read_csv(self.pfam_puncta, sep='\t', index_col=0)
        print(df['pfamA_acc'].value_counts())

    def get_segment_scores(self, segment_seq):
        total_motifs = 0
        total_length = 0
        for seq in segment_seq:
            motifs = tools_lc.count_lc_motifs(seq, self.k, self.lca, self.lce)
            total_motifs += motifs
            total_length += len(seq)
        return total_motifs, total_length

    def norm_function(self, raw_scores, lengths):
        norm_scores = []
        for raw_score, length in zip(raw_scores, lengths):
            norm_score = raw_score - ((self.lc_m * length) + self.lc_b)
            norm_scores.append(norm_score)
        return norm_scores

def main():
    rp = RemovePfam()
    rp.pfam_in_common()

if __name__ == '__main__':
    main()


#!/usr/bin/env python

"""
https://www.codewars.com/kata/520b9d2ad5c005041100000f/python
"""
import ipdb
import pytest

"""
pig_it('Pig latin is cool') # igPay atinlay siay oolcay
pig_it('Hello world !')     # elloHay orldway !
"""

# from codewars solution
def pig_it(text):
    lst = text.split()
    return ' '.join( [word[1:] + word[:1] + 'ay' if word.isalpha() else word for word in lst])

@pytest.mark.parametrize("input_, expected",
                         [('Pig latin is cool', 'igPay atinlay siay oolcay'),
                          ('Hello world !', 'elloHay orldway !')])
def test_solution(input_, expected):
    assert move_first_letter_of_each_word_to_the_end(input_) == expected

def move_first_letter_of_each_word_to_the_end(text: str)-> str:
    return ' '.join([f"{i[1:]}{i[0]}ay" if i not in ['!',',','?'] else i for i in text.split() ])
    # out_str= ''
    # for i in s.split():
    #     #print(i)
    #     out_str += i[1:]
    # return out_str

if __name__ == '__main__':
    print(move_first_letter_of_each_word_to_the_end('Pig latin is cool'))
    print(move_first_letter_of_each_word_to_the_end('Hello world !'))


'''

概念：一种保存数据的格式
作用：可以保存本地的json文件，也可以将json串进行传输，通常将json称为轻量级的传输方式

json文件组成
{}  代表对象(字典)
[]  代表列表
:   代表键值对
,   分隔两个部分

'''
import json

jsonStr = '''{
		"rate": "8.0",
		"cover_x": 1400,
		"title": "我是余欢水",
		"url": "https:\/\/movie.douban.com\/subject\/33442331\/",
		"playable": true,
		"cover": "https://img3.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2574916002.jpg",
		"id": "33442331",
		"cover_y": 2139,
		"is_new": false
	}
'''

#将json格式的字符串转为python数据类型的对象
#将json格式数据转换为字典，方便取值
jsonData = json.loads(jsonStr)
print(jsonData)
print(type(jsonData))
print(type(jsonStr))

jsonData2 = {
		"rate": "8.0",
		"cover_x": 1400,
		"title": "我是余欢水",
		"url": "https:\/\/movie.douban.com\/subject\/33442331\/",
		"playable": True,
		"cover": "https://img3.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2574916002.jpg",
		"id": "33442331",
		"cover_y": 2139,
		"is_new": False
	}

#将python数据类型的对象转换为json格式的字符串
jsonStr2 = json.dumps(jsonData2)
print(jsonStr2)
print(type(jsonData2))
print(type(jsonStr2))

正则表达式与爬虫/3、爬虫/7、json数据解析.py


from motor import Motor
import keyboard, time, sys
from threading import Thread
import tkinter as tk

'''
#PINOS DE COMUNICAÇÃO EM BINÁRIO
#[0,1,2,3, 4, 5, 6,  7] - BITS DA PLACA
#[1,2,4,8,16,32,64,128] - SINAL DE COMUNICAÇÃO CORRESPONDENTE
'''

mx = Motor(4, 8)
my = Motor(16, 32)

dx, dy = 2700, 270

while(my.pos <= 2700):
    mx.andar(dx, 1)
    if(mx.exit == True):
        break
    my.andar(dy, 1)
    if(my.exit == True):
        break
    mx.andar(dx, 0)
    if(mx.exit == True):
        break
    my.andar(dy, 1)
    if(my.exit == True):
        break

print(mx.pos, my.pos)
print('cabou')


# MIT License
#
# Copyright (c) 2023 Andrey Zhdanov (rivitna)
# https://github.com/rivitna
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import sys
import io
import struct
import zlib

MARKER = b'\xFE\x09\x00\x00\x8D'

def decompress_data(data):
    """Decompress data"""
    decompress = zlib.decompressobj(-zlib.MAX_WBITS)
    inflated = decompress.decompress(data)
    inflated += decompress.flush()
    return inflated

#
# Main
#
if len(sys.argv) != 2:
    print('Usage: '+ sys.argv[0] + ' filename')
    sys.exit(0)

filename = sys.argv[1]

with io.open(filename, 'rb') as f:
    file_data = f.read()

pos = 0

# Find configuration data
while True:

    pos = file_data.find(MARKER, pos)
    if pos < 0:
        break
    pos += len(MARKER)

    # stsfld
    if file_data[pos + 4] != 0x80:
        continue
    cfg_data_token, = struct.unpack_from('<L', file_data, pos + 5)

    # ldsfld
    if file_data[pos + 9] != 0x7E:
        continue
    token, = struct.unpack_from('<L', file_data, pos + 10)
    if token == cfg_data_token:
        pos += 9
        break

if pos < 0:
    print('Error: Configuration data not found.')
    sys.exit(1)

print('cfg data position: %08X' % pos)
print('cfg data token: 0x%08X' % cfg_data_token)

cfg_data_dict = {}

# Parse IL code
while pos + 16 <= len(file_data):

    # ldsfld
    if file_data[pos] != 0x7E:
        break
    pos += 1
    token, = struct.unpack_from('<L', file_data, pos)
    if token != cfg_data_token:
        break
    pos += 4

    # ldc.i4
    if file_data[pos] != 0x20:
        break
    pos += 1
    idx, = struct.unpack_from('<L', file_data, pos)
    if cfg_data_dict.get(idx) is not None:
        break
    pos += 4

    # ldc.i4, stelem.i1
    if (file_data[pos] != 0x20) or (file_data[pos + 5] != 0x9C):
        break
    pos += 1
    val, = struct.unpack_from('<L', file_data, pos)
    if val > 255:
        break
    pos += 5

    cfg_data_dict[idx] = val

    # skip nop
    if file_data[pos] == 0:
        pos += 1

pack_cfg_data_size = max(cfg_data_dict.keys()) + 1
print('compressed cfg data size: %d' % pack_cfg_data_size)

pack_cfg_data = b''
for i in range(pack_cfg_data_size):
    val = cfg_data_dict.get(i)
    if val is None:
        break
    pack_cfg_data += bytes([val])

cfg_data = decompress_data(pack_cfg_data)
print('cfg data size: %d' % len(cfg_data))

cfg_filename = filename + '.cfg'
with io.open(cfg_filename, 'wb') as f:
    f.write(cfg_data)


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json

# ---
# title: hello,hikyson
# tags: [Default]
# category: [Default]
# comments: true
# date: 2014-04-20 22:18:43
# ---
#
# hello,hikyson
#
# <!-- more -->
#
# |Version|Codename|API|Distribution|
# |---|---|---|---|
# |111|222|333|444|
import os

from ScrapyForAndroidDashboard.git_pusher import post_title, local_time_str, post_name, push, post_file_dir

class ScrapyforandroiddashboardPipeline(object):
    def process_item(self, item, spider):
        # generate md file
        divider = "---"
        line_feed = "\r\n"
        title = post_title
        tags = "[android,spider,scrapy]"
        category = "[scrapy]"
        comments = "true"
        date = local_time_str
        more = "<!-- more -->"
        head = "".join(
            [divider, line_feed, "title: ", title, line_feed, "tags: ", tags, line_feed, "category: ", category,
             line_feed, "comments: ", comments, line_feed, "date: ", date, line_feed, divider, line_feed])
        summary = "This is a post generate by a spider , grab from url: [developer.android.google.cn](developer.android.google.cn)"
        updatetime = "Update time: %s" % local_time_str
        version_data_dict = json.loads(item["version_data"])
        version_chart_url = "https:" + version_data_dict["chart"] + ".png"
        # version text
        text_version = "".join(
            ["![version_chart](%s)" % version_chart_url, line_feed, line_feed, "|Codename|API|Distribution|",
             line_feed, "|---|---|---|", line_feed])
        version_items = version_data_dict["data"]
        for version_item in version_items:
            api = version_item["api"]
            name = version_item["name"]
            perc = version_item["perc"]
            text_version = text_version + "|" + str(api) + "|" + name + "|" + str(perc) + "|" + line_feed

        post = "".join(
            [head, line_feed, line_feed, summary, line_feed, updatetime, line_feed, line_feed, more, line_feed,
             line_feed, text_version])
        for file_name in os.listdir(post_file_dir):
            if file_name.find(post_title) >= 0:
                os.remove(os.path.join(post_file_dir, file_name))
        file_name = os.path.join(post_file_dir, post_name)
        with open(file_name, 'wb') as f:
            f.write(post)
        push()
        return item


# https://cloud.google.com/pubsub/docs/create-topic#create_a_topic
# https://cloud.google.com/python/docs/reference/pubsub/latest

# %%
from google.cloud import pubsub_v1

# TODO(developer)
project_id = "podact-topic-extractor"
topic_id = "your-topic-id"

publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_id)

topic = publisher.create_topic(request={"name": topic_path})

print(f"Created topic: {topic.name}")
# %%
# When you delete a topic, its subscriptions are not deleted.
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_id)

publisher.delete_topic(request={"topic": topic_path})

print(f"Topic deleted: {topic_path}")
# %%
from google.cloud import pubsub_v1

publisher = pubsub_v1.PublisherClient()
project_path = f"projects/{project_id}"

for topic in publisher.list_topics(request={"project": project_path}):
    print(topic)
# %%
"""Publishes multiple messages to a Pub/Sub topic with an error handler."""
from concurrent import futures
from typing import Callable

from google.cloud import pubsub_v1

publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_id)
publish_futures = []

def get_callback(
    publish_future: pubsub_v1.publisher.futures.Future, data: str
) -> Callable[[pubsub_v1.publisher.futures.Future], None]:
    def callback(publish_future: pubsub_v1.publisher.futures.Future) -> None:
        try:
            # Wait 60 seconds for the publish call to succeed.
            print(publish_future.result(timeout=60))
        except futures.TimeoutError:
            print(f"Publishing {data} timed out.")

    return callback

for i in range(10):
    data = str(i)
    # When you publish a message, the client returns a future.
    publish_future = publisher.publish(topic_path, data.encode("utf-8"))
    # Non-blocking. Publish failures are handled in the callback function.
    publish_future.add_done_callback(get_callback(publish_future, data))
    publish_futures.append(publish_future)

# Wait for all the publish futures to resolve before exiting.
futures.wait(publish_futures, return_when=futures.ALL_COMPLETED)

print(f"Published messages with error handler to {topic_path}.")
# %%
from google.cloud import pubsub_v1

# TODO(developer)
# project_id = "your-project-id"
# topic_id = "your-topic-id"

publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_id)

for n in range(1, 10):
    data_str = f"Message number {n}"
    # Data must be a bytestring
    data = data_str.encode("utf-8")
    # Add two attributes, origin and username, to the message
    future = publisher.publish(
        topic_path, data, origin="python-sample", username="gcp"
    )
    print(future.result())

print(f"Published messages with custom attributes to {topic_path}.")
# %%
from google.cloud import pubsub_v1

# TODO(developer): Choose an existing topic.
# project_id = "your-project-id"
# topic_id = "your-topic-id"

publisher_options = pubsub_v1.types.PublisherOptions(enable_message_ordering=True)
# Sending messages to the same region ensures they are received in order
# even when multiple publishers are used.
client_options = {"api_endpoint": "us-east1-pubsub.googleapis.com:443"}
publisher = pubsub_v1.PublisherClient(
    publisher_options=publisher_options, client_options=client_options
)
# The `topic_path` method creates a fully qualified identifier
# in the form `projects/{project_id}/topics/{topic_id}`
topic_path = publisher.topic_path(project_id, topic_id)

for message in [
    ("message1", "key1"),
    ("message2", "key2"),
    ("message3", "key1"),
    ("message4", "key2"),
]:
    # Data must be a bytestring
    data = message[0].encode("utf-8")
    ordering_key = message[1]
    # When you publish a message, the client returns a future.
    future = publisher.publish(topic_path, data=data, ordering_key=ordering_key)
    print(future.result())

print(f"Published messages with ordering keys to {topic_path}.")
# %%
from concurrent import futures

from google.cloud import pubsub_v1

# TODO(developer)
# project_id = "your-project-id"
# topic_id = "your-topic-id"

# Configure the batch to publish as soon as there are 10 messages
# or 1 KiB of data, or 1 second has passed.
batch_settings = pubsub_v1.types.BatchSettings(
    max_messages=10,  # default 100
    max_bytes=1024,  # default 1 MB
    max_latency=1,  # default 10 ms
)
publisher = pubsub_v1.PublisherClient(batch_settings)
topic_path = publisher.topic_path(project_id, topic_id)
publish_futures = []

# Resolve the publish future in a separate thread.
def callback(future: pubsub_v1.publisher.futures.Future) -> None:
    message_id = future.result()
    print(message_id)

for n in range(1, 10):
    data_str = f"Message number {n}"
    # Data must be a bytestring
    data = data_str.encode("utf-8")
    publish_future = publisher.publish(topic_path, data)
    # Non-blocking. Allow the publisher client to batch multiple messages.
    publish_future.add_done_callback(callback)
    publish_futures.append(publish_future)

futures.wait(publish_futures, return_when=futures.ALL_COMPLETED)

print(f"Published messages with batch settings to {topic_path}.")
# %%
from google import api_core
from google.cloud import pubsub_v1

# %%
# TODO(developer)
# project_id = "your-project-id"
# topic_id = "your-topic-id"

# Configure the retry settings. Defaults shown in comments are values applied
# by the library by default, instead of default values in the Retry object.
custom_retry = api_core.retry.Retry(
    initial=0.250,  # seconds (default: 0.1)
    maximum=90.0,  # seconds (default: 60.0)
    multiplier=1.45,  # default: 1.3
    deadline=300.0,  # seconds (default: 60.0)
    predicate=api_core.retry.if_exception_type(
        api_core.exceptions.Aborted,
        api_core.exceptions.DeadlineExceeded,
        api_core.exceptions.InternalServerError,
        api_core.exceptions.ResourceExhausted,
        api_core.exceptions.ServiceUnavailable,
        api_core.exceptions.Unknown,
        api_core.exceptions.Cancelled,
    ),
)

publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_id)

for n in range(1, 10):
    data_str = f"Message number {n}"
    # Data must be a bytestring
    data = data_str.encode("utf-8")
    future = publisher.publish(topic=topic_path, data=data, retry=custom_retry)
    print(future.result())

print(f"Published messages with retry settings to {topic_path}.")


#!usr/bin/env python
# -*- coding:utf-8 -*-
"""
@author: admin
@file: MultiHeadedAttention.py
@time: 2021/09/02
@desc:
"""
import copy
import torch
import math
from torch import nn
import torch.nn.functional as F

def clones(module, N):
    """
    克隆基本单元，克隆的单元之间参数不共享
    """
    return nn.ModuleList([
        copy.deepcopy(module) for _ in range(N)
    ])

def attention(query, key, value, mask=None, dropout=None):
    """
    Scaled Dot-Product Attention（方程（4））
    """
    # q、k、v向量长度为d_k
    d_k = query.size(-1)
    # 矩阵乘法实现q、k点积注意力，sqrt(d_k)归一化
    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)
    # 注意力掩码机制
    if mask is not None:
        scores = scores.masked_fill(mask==0, -1e9)
    # 注意力矩阵softmax归一化
    p_attn = F.softmax(scores, dim=-1)
    # dropout
    if dropout is not None:
        p_attn = dropout(p_attn)
    # 注意力对v加权
    return torch.matmul(p_attn, value), p_attn

class MultiHeadedAttention(nn.Module):
    """
    Multi-Head Attention（编码器（2））
    """
    def __init__(self, h, d_model, dropout=0.1):
        super(MultiHeadedAttention, self).__init__()
        """
        `h`：注意力头的数量
        `d_model`：词向量维数
        """
        # 确保整除
        assert d_model % h == 0
        # q、k、v向量维数
        self.d_k = d_model // h
        # 头的数量
        self.h = h
        # WQ、WK、WV矩阵及多头注意力拼接变换矩阵WO
        self.linears = clones(nn.Linear(d_model, d_model), 4)
        self.attn = None
        self.dropout = nn.Dropout(p=dropout)

    def forward(self, query, key, value, mask=None):
        if mask is not None:
            mask = mask.unsqueeze(1)
        # 批次大小
        nbatches = query.size(0)
        # WQ、WK、WV分别对词向量线性变换，并将结果拆成h块
        query, key, value = [
            l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)
            for l, x in zip(self.linears, (query, key, value))
        ]
        # 注意力加权
        x, self.attn = attention(query, key, value, mask=mask, dropout=self.dropout)
        # 多头注意力加权拼接
        x = x.transpose(1, 2).contiguous().view(nbatches, -1, self.h * self.d_k)
        # 对多头注意力加权拼接结果线性变换
        return self.linears[-1](x)


import pandas as pd
import plotly.express as px

from ..sequence_info.sequences import group_2_coins
from ..utils import google_form_question_to_coin_sequence

DATA_FILENAME = "C:/Users/Crystal Wang/Downloads/9.660/9.660-final-project/data/data.csv"

def get_df():
    df = pd.read_csv(DATA_FILENAME)
    df = df.drop("Timestamp", axis = 1)

    # Get rid of the music data
    df = df.drop(df.columns[-70:], axis = 1)

    # Clean ratings
    df = df.replace("1 (least representative)", 1)
    df = df.replace("7 (most representative)", 7)

    # Set index
    df = df.set_index("Name")
    return df

def split_groups(df):
    group1_df = df[df["Who asked you to do this survey?"] == "Crystal (Group 1)"]
    group1_df = group1_df.drop(group1_df.columns[-37:], axis = 1)
    group1_df = group1_df.drop("Who asked you to do this survey?", axis = 1)

    group2_df = df[df["Who asked you to do this survey?"] == "Julia (Group 2)"]
    group2_df = group2_df.drop(group2_df.columns[-73: -36], axis = 1)
    group2_df = group2_df.drop("Who asked you to do this survey?", axis = 1)
    return group1_df, group2_df

def get_control_data(g_df):
    g_control_df = g_df.drop(g_df.columns[5:], axis = 1)
    g_control_df = g_control_df.astype(int)
    g_control_df.columns = [1,2,3,4,5]
    return g_control_df

def plot_line_data(df):
    df = df.stack().reset_index()
    df = df.rename(columns = {"level_1": "x", 0: "y"})
    px.line(df, x = "x", y = "y", color = "Name")

def significance_t_test(df1, df2):
    df1_means = df1.mean(axis = 0)
    df2_means = df2.mean(axis = 0)

    df1_vars = df1.std(axis = 0, ddof = 0) ** 2
    df2_vars = df2.std(axis = 0, ddof = 0) ** 2

    df1_vars /= len(df1.index)
    df2_vars /= len(df2.index)

    numerator = df1_means - df2_means
    denominator = (df1_vars + df2_vars) ** 0.5

    return (numerator / denominator).abs() > 1.96

def verify_control_significance(g1_df, g2_df, plot = False):
    g1_control_df = get_control_data(g1_df)
    g2_control_df = get_control_data(g2_df)

    if plot:
        plot_line_data(g1_control_df)
        plot_line_data(g2_control_df)

    # See if there is a significant difference for each sequence
    is_significant = significance_t_test(g1_control_df, g2_control_df)

    if is_significant.any():
        raise Exception("The samples are significantly different in control responses!")
    else:
        print("No significant difference between the control responses.\n")

    return

def remove_control_data(g_df):
    return g_df.drop(g_df.columns[:5], axis = 1).astype(int)

def sort_columns(df):
    return df.reindex(sorted(df.columns), axis=1)

def clean_group_1(g1_df):
    g1_df.columns = [google_form_question_to_coin_sequence(column) for column in g1_df.columns]
    g1_df = sort_columns(g1_df)
    return g1_df

def clean_group_2(g2_df):
    g2_df.columns = [
        google_form_question_to_coin_sequence(f"{column} [Coin {coin}: ]")
        for column, coin in zip(g2_df.columns, group_2_coins)
    ]
    g2_df = sort_columns(g2_df)
    return g2_df

def test_experiment_significance(g1_df, g2_df):
    print("---- GROUP 1 vs. GROUP 2 EXPERIMENT ----")
    is_significant = significance_t_test(g1_df, g2_df)

    if is_significant.any():
        significant_cols = list((is_significant.loc[is_significant]).index)
        print(f"There is a significant difference between group 1 and group 2: {significant_cols}\n")

    else:
        print("There is NO significant difference between group 1 and group 2\n")

def get_groups():
    df = get_df()
    g1_df, g2_df = split_groups(df)

    verify_control_significance(g1_df, g2_df)

    g1_df = remove_control_data(g1_df)
    g2_df = remove_control_data(g2_df)

    g1_df = clean_group_1(g1_df)
    g2_df = clean_group_2(g2_df)

    test_experiment_significance(g1_df, g2_df)

    return g1_df, g2_df


#!/usr/bin/env python
# coding: utf-8

# In[12]:

# Question 1 c)
# Author: Ilyas Sharif

import numpy as np
import matplotlib.pyplot as plt

# Defining the parameters that didn't change (same as code for before)
v_f = 0.1
omega_0 = 1
tau = 1
gamma = 0.5
a = 0.0
b = 100.0
N = 10000
h = (b-a)/N
tpoints = np.arange(a, b,h)
x_0 = 0
y_0 = 0

# Defining the xpoints and ypoints array.
vp = v_f * np.log((gamma*tau)/v_f)
v_p = [0.1*vp,0.25*vp, 0.5*vp, 0.75*vp, 1*vp, 1.25*vp, 1.5*vp]
indexing = [0.1,0.25,0.5,0.75,1,1.25,1.5]

for i in range(len(v_p)):
    C = v_p[i]
    xpoints = []
    r = np.array([x_0, y_0], float)
    # Creating f(r,t) where r = (x, y = dx/dt)
    def f(r,t):
        x = r[0]
        y = r[1]
        fx = y
        fy = -(omega_0**2)*((x) - v_p[i] * t) - (y)/tau - gamma*np.exp(-np.abs(y)/v_f)
        return np.array([fx, fy], float)

    # Creating r array and computing RK4 method (copied from Newman odesim.py)
    for t in tpoints:
        xpoints.append(r[0])
        k1 = h*f(r,t)
        k2 = h*f(r+0.5*k1,t+0.5*h)
        k3 = h*f(r+0.5*k2,t+0.5*h)
        k4 = h*f(r+k3,t+h)
        r += (k1+2*k2+2*k3+k4)/6
    plt.plot(tpoints, xpoints, label = '$v_p$ = ' + str(indexing[i]) + '$v_p$')
    # I'm going to comment this out, but if you want to see the constant velocity
    # solutions that each of them oscillate around, feel free to comment out the
    # 2 lines below :)
    #x0 = -(1/omega_0**2)*(C/tau + gamma*np.exp(-C/v_f)) + (v_p[i]*tpoints)
    #plt.plot(tpoints,x0, linestyle = ":", color = 'k')

plt.title("Comparison of Different Choices for $v_p$")
plt.xlabel(" time (seconds) ")
plt.ylabel(" position (meters) ")
plt.legend()
plt.xlim(0,100)
plt.show()

# In[ ]:

Differentiation and Differential Equations/Lab06_Q1_c.py


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

__version__ = "0.1.2"
__author__ = "Abien Fred Agarap"

import argparse
from models.svm.svm import Svm

# Hyper-parameters
BATCH_SIZE = 256
LEARNING_RATE = 1e-5
N_CLASSES = 2
SEQUENCE_LENGTH = 21

def parse_args():
    parser = argparse.ArgumentParser(description="SVM for Intrusion Detection")
    group = parser.add_argument_group("Arguments")
    group.add_argument(
        "-o",
        "--operation",
        required=True,
        type=str,
        help='the operation to perform: "train" or "test"',
    )
    group.add_argument(
        "-t",
        "--train_dataset",
        required=False,
        type=str,
        help="the NumPy array training dataset (*.npy) to be used",
    )
    group.add_argument(
        "-v",
        "--validation_dataset",
        required=True,
        type=str,
        help="the NumPy array validation dataset (*.npy) to be used",
    )
    group.add_argument(
        "-c",
        "--checkpoint_path",
        required=True,
        type=str,
        help="path where to save the trained model",
    )
    group.add_argument(
        "-l",
        "--log_path",
        required=False,
        type=str,
        help="path where to save the TensorBoard logs",
    )
    group.add_argument(
        "-m",
        "--model_name",
        required=False,
        type=str,
        help="filename for the trained model",
    )
    group.add_argument(
        "-r",
        "--result_path",
        required=True,
        type=str,
        help="path where to save the actual and predicted labels",
    )
    arguments = parser.parse_args()
    return arguments

def main(arguments):

    if arguments.operation == "train":
        train_features, train_labels = data.load_data(dataset=arguments.train_dataset)
        validation_features, validation_labels = data.load_data(
            dataset=arguments.validation_dataset
        )

        train_size = train_features.shape[0]
        validation_size = validation_features.shape[0]

        model = Svm(
            alpha=LEARNING_RATE,
            batch_size=BATCH_SIZE,
            svm_c=arguments.svm_c,
            num_classes=N_CLASSES,
            num_features=SEQUENCE_LENGTH,
        )

        model.train(
            checkpoint_path=arguments.checkpoint_path,
            log_path=arguments.log_path,
            model_name=arguments.model_name,
            epochs=arguments.num_epochs,
            result_path=arguments.result_path,
            train_data=[train_features, train_labels],
            train_size=train_size,
            validation_data=[validation_features, validation_labels],
            validation_size=validation_size,
        )
    elif arguments.operation == "test":
        test_features, test_labels = data.load_data(
            dataset=arguments.validation_dataset
        )

        test_size = test_features.shape[0]

        test_features = test_features[: test_size - (test_size % BATCH_SIZE)]
        test_labels = test_labels[: test_size - (test_size % BATCH_SIZE)]

        test_size = test_features.shape[0]

        Svm.predict(
            batch_size=BATCH_SIZE,
            num_classes=N_CLASSES,
            test_data=[test_features, test_labels],
            test_size=test_size,
            checkpoint_path=arguments.checkpoint_path,
            result_path=arguments.result_path,
        )

if __name__ == "__main__":
    args = parse_args()

    main(args)


import os
import re
import sys

from lib.instruction import Instruction, AInstruction, CInstruction, LInstruction
from typing import Generator, Tuple

class Parser:
    """
    Parse the Xxx.asm into stream of instructions.
    - read source file
    - understand the format of input file
    - break each into different components
        - C: dest, comp, jump
        - A: value
    """

    def __init__(self, path: str):
        self._path = path

    def _get_clean_line(self, line):
        return self.strip_comments(line.strip())

    def _parse_c_instruction(self, line):
        if line.count(";") > 1:
            raise ValueError('line format error, should not have more than one ";" ')

        if line.count("=") > 1:
            raise ValueError('line format error, should not have more than one "="')

        if "=" in line and ";" in line:
            # D=M; JMP
            dest, tail = line.split("=")
            comp, jump = tail.split(";")
            return CInstruction(dest=dest.strip(), comp=comp.strip(), jump=jump.strip())
        elif ";" in line:
            # M; JMP
            comp, jump = line.split(";")
            return CInstruction(dest=None, comp=comp.strip(), jump=jump.strip())
        elif "=" in line:
            # M=D
            dest, comp = line.split("=")
            return CInstruction(dest=dest.strip(), comp=comp.strip(), jump=None)
        else:
            # D
            return CInstruction(dest=None, comp=line.strip(), jump=None)

        raise ValueError("line format invalid: ", line)

    def _parse(self, line) -> Instruction:
        if line.startswith("("):
            inst = LInstruction(name=line[1:-1].strip())
        elif line.startswith("@"):
            inst = AInstruction(value=line[1:])
        else:
            inst = self._parse_c_instruction(line)

        return inst

    def get_instruction(self) -> Generator[Tuple[int, Instruction], None, None]:
        with open(self._path, "r") as f:
            for line in f:
                line = self._get_clean_line(line)
                if not line:
                    continue
                instruction = self._parse(line)
                yield instruction

    def strip_comments(self, text):
        return re.sub(
            "//.*?$|/\*.*?\*/|'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"",
            "",
            text,
            flags=re.S,
        ).strip()


import cv2
import numpy
import torch
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import KFold
import os
import random
from tqdm import tqdm
from segment_anything import SamAutomaticMaskGenerator, sam_model_registry, SamPredictor
import argparse
from utils.utils import *
import time

def get_embedding(img, predictor):
    predictor.set_image(img)
    img_emb = predictor.get_image_embedding()
    return img_emb

def train(args, predictor):
    data_path = args.data_path
    assert os.path.exists(data_path), 'data path does not exist!'

    num_image = args.k

    fnames = os.listdir(os.path.join(data_path, 'images'))
    # get 20 random indices from fnames
    random.shuffle(fnames)
    fnames = fnames[:num_image]
    image_embeddings = []
    labels = []

    # get the image embeddings
    print('Start training...')
    t1 = time.time()
    i = 0
    for fname in tqdm(fnames):
        # read data
        image = cv2.imread(os.path.join(data_path, 'images', fname))
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        mask = cv2.imread(os.path.join(data_path, 'masks', fname))
        mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
        _, mask = cv2.threshold(mask, 128, 1, cv2.THRESH_BINARY) # threshold the mask to 0 and 1
        downsampled_mask = cv2.resize(mask, dsize=(64, 64), interpolation=cv2.INTER_NEAREST)

        img_emb = get_embedding(image, predictor)
        img_emb = img_emb.cpu().numpy().transpose((2, 3, 1, 0)).reshape((64, 64, 256)).reshape(-1, 256)
        image_embeddings.append(img_emb)

        labels.append(downsampled_mask.flatten())

        i += 1
        if i > num_image: break
    t2 = time.time()
    print("Time used: {}m {}s".format((t2 - t1) // 60, (t2 - t1) % 60))
    image_embeddings_cat = np.concatenate(image_embeddings)
    labels = np.concatenate(labels)

    # Create a linear regression model and fit it to the training data
    model = LogisticRegression(max_iter=1000)
    model.fit(image_embeddings_cat, labels)

    return model

def test_visualize(args, model, predictor):
    data_path = args.data_path

    num_image = args.k
    fnames = os.listdir(os.path.join(data_path, 'images'))
    random.shuffle(fnames)
    fnames = fnames[num_image:]
    num_visualize = args.visualize_num

    dice_linear = []
    dice1 = []
    dice2 = []
    dice3 = []
    i = 0

    for fname in tqdm(fnames[:num_visualize]):
        # read data
        image = cv2.imread(os.path.join(data_path, 'images', fname))
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        mask = cv2.imread(os.path.join(data_path, 'masks', fname))
        mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
        _, mask = cv2.threshold(mask, 128, 1, cv2.THRESH_BINARY)
        H, W, _ = image.shape

        # get the image embedding and flatten it
        img_emb = get_embedding(image, predictor)
        img_emb = img_emb.cpu().numpy().transpose((2, 3, 1, 0)).reshape((64, 64, 256)).reshape(-1, 256)

        # get the mask predicted by the linear classifier
        y_pred = model.predict(img_emb)
        y_pred = y_pred.reshape((64, 64))
        # mask predicted by the linear classifier
        mask_pred_l = cv2.resize(y_pred, (mask.shape[1], mask.shape[0]), interpolation=cv2.INTER_NEAREST)

        # use distance transform to find a point inside the mask
        fg_point = get_max_dist_point(mask_pred_l)
        # Define the kernel for dilation
        kernel = np.ones((5, 5), np.uint8)
        eroded_mask = cv2.erode(mask_pred_l, kernel, iterations=3)
        mask_pred_l = cv2.dilate(eroded_mask, kernel, iterations=5)

        # set the image to sam
        predictor.set_image(image)

        # prompt the sam with the point
        input_point = np.array([[fg_point[0], fg_point[1]]])
        input_label = np.array([1])
        masks_pred_sam_prompted1, _, _ = predictor.predict(
            point_coords=input_point,
            point_labels=input_label,
            box=None,
            multimask_output=False,
        )

        # prompt the sam with the bounding box
        y_indices, x_indices = np.where(mask_pred_l > 0)
        if np.all(mask_pred_l == 0):
            bbox = np.array([0, 0, H, W])
        else:
            x_min, x_max = np.min(x_indices), np.max(x_indices)
            y_min, y_max = np.min(y_indices), np.max(y_indices)
            H, W = mask_pred_l.shape
            x_min = max(0, x_min - np.random.randint(0, 20))
            x_max = min(W, x_max + np.random.randint(0, 20))
            y_min = max(0, y_min - np.random.randint(0, 20))
            y_max = min(H, y_max + np.random.randint(0, 20))
            bbox = np.array([x_min, y_min, x_max, y_max])
        masks_pred_sam_prompted2, _, _ = predictor.predict(
            point_coords=None,
            point_labels=None,
            box=bbox[None, :],
            multimask_output=False,)

        # prompt the sam with both the point and bounding box
        masks_pred_sam_prompted3, _, _ = predictor.predict(
            point_coords=input_point,
            point_labels=input_label,
            box=bbox[None, :],
            multimask_output=False,)

        dice_l = dice_coef(mask, mask_pred_l)
        dice_p = dice_coef(mask, masks_pred_sam_prompted1[0])
        dice_b = dice_coef(mask, masks_pred_sam_prompted2[0])
        dice_i = dice_coef(mask, masks_pred_sam_prompted3[0])
        dice_linear.append(dice_l)
        dice1.append(dice_p)
        dice2.append(dice_b)
        dice3.append(dice_i)

        # plot the results
        fig, ax = plt.subplots(1, 5, figsize=(15, 10))
        ax[0].set_title('Ground Truth')
        ax[0].imshow(mask)
        ax[1].set_title('Linear + e&d')
        ax[1].plot(fg_point[0], fg_point[1], 'r.')
        ax[1].imshow(mask_pred_l)
        ax[2].set_title('Point')
        ax[2].plot(fg_point[0], fg_point[1], 'r.')
        ax[2].imshow(masks_pred_sam_prompted1[0])
        ax[3].set_title('Box')
        show_box(bbox, ax[3])
        ax[3].imshow(masks_pred_sam_prompted2[0])
        ax[4].set_title('Point + Box')
        ax[4].plot(fg_point[0], fg_point[1], 'r.')
        show_box(bbox, ax[4])
        ax[4].imshow(masks_pred_sam_prompted3[0])
        [axi.set_axis_off() for axi in ax.ravel()]

        if os.path.exists(args.save_path) == False:
            os.mkdir(args.save_path)
        plt.savefig(os.path.join(args.save_path, fname.split('.')[0]+str(i)))

    mdice0 = round(sum(dice_linear)/float(len(dice_linear)), 5)
    mdice1 = round(sum(dice1)/float(len(dice1)), 5)
    mdice2 = round(sum(dice2)/float(len(dice2)), 5)
    mdice3 = round(sum(dice3)/float(len(dice3)), 5)

    print('For the first {} images: '.format(num_visualize))
    print('mdice(linear classifier: )', mdice0)
    print('mDice(point prompts): ', mdice1)
    print('mDice(bbox prompts): ', mdice2)
    print('mDice(points and boxes): ', mdice3)

def test(args, predictor):
    data_path = args.data_path
    images = []
    masks = []
    fnames = os.listdir(os.path.join(data_path, 'images'))
    print(f'loading images from {data_path}...')
    for fname in tqdm(fnames):
        # read data
        image = cv2.imread(os.path.join(data_path, 'images', fname))
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        mask = cv2.imread(os.path.join(data_path, 'masks', fname))
        mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
        _, mask = cv2.threshold(mask, 128, 1, cv2.THRESH_BINARY)
        images.append(image)
        masks.append(mask)

    kf = KFold(n_splits=5, shuffle=True, random_state=42)
    for train_index, text_index in kf.split(images):
        train_images = [images[i] for i in train_index]
        train_masks = [masks[i] for i in train_index]
        test_images = [images[i] for i in text_index]
        test_masks = [masks[i] for i in text_index]

        # train the linear classifier
        k = args.k
        random_indices = random.sample(range(len(train_images)), k)
        image_embeddings = []
        labels = []
        for idx in random_indices:
            image = train_images[idx]
            mask = train_masks[idx]
            downsampled_mask = cv2.resize(mask, dsize=(64, 64), interpolation=cv2.INTER_NEAREST)

            img_emb = get_embedding(image)
            img_emb = img_emb.cpu().numpy().transpose((2, 3, 1, 0)).reshape((64, 64, 256)).reshape(-1, 256)
            image_embeddings.append(img_emb)
            labels.append(downsampled_mask.flatten())

        image_embeddings_cat = numpy.concatenate(image_embeddings)
        labels = numpy.concatenate(labels)

        model = LogisticRegression(max_iter=1000) # how to set parameters?? C, max_iter, verbose, solver
        model.fit(image_embeddings_cat, labels)

        # test
        dice_linear=[]
        dice1=[]
        dice2=[]
        dice3=[]
        for idx in range(len(test_images)):
            image = test_images[idx]
            mask = test_masks[idx]
            H, W, _ = image.shape

            img_emb = get_embedding(image)
            img_emb = img_emb.cpu().numpy().transpose((2, 3, 1, 0)).reshape((64, 64, 256)).reshape(-1, 256)

            # ger the mask predicted by the linear classifier
            y_pred = model.predict(img_emb)
            y_pred = y_pred.reshape((64,64))
            mask_pred_l = cv2.resize(y_pred, (mask.shape[1], mask.shape[0]), interpolation=cv2.INTER_NEAREST)

            # use distance transform to find a point inside the mask
            fg_point = get_max_dist_point(mask_pred_l)

            # Define the kernel for dilation
            kernel = np.ones((5, 5), np.uint8)

            eroded_mask = cv2.erode(mask_pred_l, kernel, iterations=3)
            mask_pred_l = cv2.dilate(eroded_mask, kernel, iterations=5)

            # set the image to sam
            predictor.set_image(image)

            # prompt sam with the point
            input_point = np.array([[fg_point[0], fg_point[1]]])
            input_label = np.array([1])
            masks_pred_sam_prompted1, _, logits = predictor.predict(
            point_coords=input_point,
            point_labels=input_label,
            box=None,
            multimask_output=False,)

            # prompt sam with the bbox
            y_indices, x_indices = np.where(mask_pred_l > 0)
            if np.all(mask_pred_l==0):
                bbox = np.array([0 ,0, H, W])
            else:
                x_min, x_max = np.min(x_indices), np.max(x_indices)
                y_min, y_max = np.min(y_indices), np.max(y_indices)
                H, W = mask_pred_l.shape
                x_min = max(0, x_min - np.random.randint(0, 20))
                x_max = min(W, x_max + np.random.randint(0, 20))
                y_min = max(0, y_min - np.random.randint(0, 20))
                y_max = min(H, y_max + np.random.randint(0, 20))
                bbox = np.array([x_min, y_min, x_max, y_max])
                masks_pred_sam_prompted2, _, _ = predictor.predict(
                point_coords=None,
                point_labels=None,
                box=bbox[None, :],
                multimask_output=False,)

                masks_pred_sam_prompted3, _, _,= predictor.predict(
                point_coords=input_point,
                point_labels=input_label,
                box=bbox[None, :],
                multimask_output=False,)

                dice_l = dice_coef(mask, mask_pred_l)
                dice_p = dice_coef(mask, masks_pred_sam_prompted1[0])
                dice_b = dice_coef(mask, masks_pred_sam_prompted2[0])
                dice_c = dice_coef(mask, masks_pred_sam_prompted3[0])
                dice_linear.append(dice_l)
                dice1.append(dice_p)
                dice2.append(dice_b)
                dice3.append(dice_c)

        mdice0 = round(sum(dice_linear)/float(len(dice_linear)), 5)
        mdice1 = round(sum(dice1)/float(len(dice1)), 5)
        mdice2 = round(sum(dice2)/float(len(dice2)), 5)
        mdice3 = round(sum(dice3)/float(len(dice3)), 5)

        print('mdice(linear classifier: )', mdice0)
        print('mDice(point prompts): ', mdice1)
        print('mDice(bbox prompts): ', mdice2)
        print('mDice(points and boxes): ', mdice3)
        print('\n')

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('--device', type=str, default='cuda:0', help='device')
    parser.add_argument('--k', type=int, default=10, help='number of pics')
    parser.add_argument('--data_path', type=str, default='./data/Kvasir-SEG', help='path to train data')
    parser.add_argument('--model_type', type=str, default='vit_b', help='SAM model type')
    parser.add_argument('--checkpoint', type=str, default='./checkpoints/sam_vit_b_01ec64.pth', help='SAM checkpoint')
    parser.add_argument('--visualize', type=bool, default=True, help='visualize the results')
    parser.add_argument('--save_path', type=str, default='./results', help='path to save the results')
    parser.add_argument('--visualize_num', type=int, default=30, help='number of pics to visualize')
    args = parser.parse_args()

    # set random seed
    random.seed(42)

    # register the SAM model
    sam = sam_model_registry[args.model_type](checkpoint=args.checkpoint).to(args.device)
    global predictor
    predictor = SamPredictor(sam)
    print('SAM model loaded!', '\n')

    if args.visualize:
        model = train(args, predictor)
        test_visualize(args, model, predictor)
    else:
        test(args, predictor)

if __name__ == '__main__':
    main()