# -*- coding: utf-8 -*-
from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes
from plone import api
from plone.app.dexterity.behaviors import constrains
from plone.app.textfield.value import RichTextValue
from zope.dottedname.resolve import resolve
from Products.Five.utilities.marker import mark
from Products.CMFPlone.interfaces import INonInstallable
from zope.interface import implementer
from plone.namedfile.file import NamedImage

from logging import getLogger
logger = getLogger(__name__)

@implementer(INonInstallable)
class HiddenProfiles(object):

    def getNonInstallableProfiles(self):
        """Hide uninstall profile from site-creation and quickinstaller"""
        return [
            'qyxycjh.policy:uninstall',
        ]

# for image field data

def _load_image(slider):
    from plone.namedfile.file import NamedImage
    import os
    filename = os.path.join(
        os.path.dirname(__file__),
        'browser',
        'static',
        'slide_{0}.jpg'.format(slider),
    )
    return NamedImage(
        data=open(filename, 'r').read(),
        filename=u'slide_{0}.jpg'.format(slider)
    )
image1 = _load_image(1)
image2 = _load_image(2)
image3 = _load_image(3)

default = { 'i': 'portal_type',
           'o': 'plone.app.querystring.operation.string.is',
            'v': 'Document'}
query = []
defaultpath = {
                    'i': 'path',
                    'o': 'plone.app.querystring.operation.string.path',
                    'v': '/',
                }
import copy

defaultpath.update({'v':'/guanyuqixie'})
guanyuqixie = [default,defaultpath]

xiehuidongtaipath = copy.copy(defaultpath)
xiehuidongtaipath.update({'v':'/xiehuidongtai'})
xiehuidongtai = [default,xiehuidongtaipath]

hangyezixunpath = copy.copy(defaultpath)
hangyezixunpath.update({'v':'/hangyezixun'})
hangyezixun = [default,hangyezixunpath]

zhengcefaguipath = copy.copy(defaultpath)
zhengcefaguipath.update({'v':'/zhengcefagui'})
zhengcefagui = [default,zhengcefaguipath]

meitibaodaopath = copy.copy(defaultpath)
meitibaodaopath.update({'v':'/meitibaodao'})
meitibaodao = [default,meitibaodaopath]

huiyuandanweipath = copy.copy(defaultpath)
huiyuandanweipath.update({'v':'/huiyuandanwei'})
huiyuandanwei = [default,huiyuandanweipath]

STRUCTURE = [
    {
        'type': 'Folder',
        'title': u'关于企协',
        'id': 'guanyuqixie',
        'description': u'关于企协',
        'layout': 'tableview'
    },
    {
        'type': 'Folder',
        'title': u'协会动态',
        'id': 'xiehuidongtai',
        'description': u'协会动态',
        'layout': 'tableview',
        'children': [
                     {
            'type': 'my315ok.products.productfolder',
            'title': u'图片新闻',
            'id': 'tupianxinwen',
            'description': u'图片新闻',
            'children': [
                     {
            'type': 'my315ok.products.product',
            'title': u'图片新闻',
            'id': 'prdt1',
            'image':image1,
            'description': u'图片新闻'
                        } ,
                         {
            'type': 'my315ok.products.product',
            'title': u'图片新闻2',
            'id': 'prdt2',
            'image':image2,
            'description': u'图片新闻2'
                        } ,
                         {
            'type': 'my315ok.products.product',
            'title': u'图片新闻3',
            'id': 'prdt3',
            'image':image3,
            'description': u'图片新闻3'
                        }
                         ]
                      }
                     ]
    },
    {
        'type': 'Folder',
        'title': u'行业资讯',
        'id': 'hangyezixun',
        'description': u'行业资讯',
        'layout': 'tableview'
    },
    {
        'type': 'Folder',
        'title': u'政策法规',
        'id': 'zhengcefagui',
        'description': u'政策法规',
        'layout': 'tableview'

    },
    {
        'type': 'Folder',
        'title': u'会员单位',
        'id': 'huiyuandanwei',
        'description': u'会员单位',
        'layout': 'tableview'

    },
    {
        'type': 'Folder',
        'title': u'媒体报道',
        'id': 'meitibaodao',
        'description': u'媒体报道',
        'layout': 'tableview'
    },
    {
        'type': 'Folder',
        'title': u'查询集',
        'id': 'sqls',
        'description': u'查询集',
        'children': [
                     {
                     'type':'Collection',
                     'title':u'协会动态',
                     'description': u'协会动态查询集',
                     'id':'xiehuidongtai',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':xiehuidongtai,
                     },
                     {
                     'type':'Collection',
                     'title':u'行业资讯',
                     'description': u'行业资讯查询集',
                     'id':'hangyezixun',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':hangyezixun,
                     },
                     {
                     'type':'Collection',
                     'title':u'会员单位',
                     'description': u'会员单位查询集',
                     'id':'huiyuandanwei',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':huiyuandanwei,
                     },
                    {
                     'type':'Collection',
                     'title':u'政策法规',
                     'description': u'最新政策法规',
                     'id':'zhengcefagui',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':zhengcefagui,
                     },
                    {
                     'type':'Collection',
                     'title':u'媒体报道',
                     'description': u'媒体报道',
                     'id':'meitibaodao',
                     'sort_on':'created',
                     'sort_reversed':True,
                     'query':meitibaodao,
                     }

                     ]
    },
    {
        'type': 'Folder',
        'title': u'帮助',
        'id': 'help',
        'description': u'帮助',
        'layout': 'tableview',
    }

]

def isNotCurrentProfile(context):
    return context.readDataFile('policy_marker.txt') is None

def post_install(context):
    """Setuphandler for the profile 'default'
    """
#     if isNotCurrentProfile(context):
#         return
    # Do something during the installation of this package
#     return
    portal = api.portal.get()
    members = portal.get('events', None)
    if members is not None:
        api.content.delete(members)
    members = portal.get('news', None)
    if members is not None:
        api.content.delete(members)
    members = portal.get('Members', None)
    if members is not None:
       members.exclude_from_nav = True
       members.reindexObject()

    for item in STRUCTURE:
        _create_content(item, portal)

    import_article(portal)
    members = portal.get('sqls', None)
    if members is not None:
       members.exclude_from_nav = True
       members.reindexObject()
    members = portal.get('help', None)
    if members is not None:
       members.exclude_from_nav = True
       members.reindexObject()

def uninstall(context):
    """Uninstall script"""
    # Do something at the end of the uninstallation of this package.

def _create_content(item, container):
    new = container.get(item['id'], None)
    if not new:

        new = api.content.create(
            type=item['type'],
            container=container,
            title=item['title'],
            description=item['description'],
            id=item['id'],
            safe_id=False)
        logger.info('Created item {}'.format(new.absolute_url()))

    if item.get('layout', False):
        new.setLayout(item['layout'])
    if item.get('query', False):
        new.query = item['query']
    if item.get('sort_on', False):
        new.sort_on = item['sort_on']
    if item.get('sort_reversed', False):
        new.sort_reversed = item['sort_reversed']
    if item.get('image', False):
        new.image = item['image']
    if item.get('markif', False):
        try:
            ifobj = resolve(item['markif'])
            mark(new,ifobj)
        except:
            pass
    if item.get('default-page', False):
        new.setDefaultPage(item['default-page'])
    if item.get('allowed_types', False):
        _constrain(new, item['allowed_types'])
    if item.get('local_roles', False):
        for local_role in item['local_roles']:
            api.group.grant_roles(
                groupname=local_role['group'],
                roles=local_role['roles'],
                obj=new)
    if item.get('publish', False):
        api.content.transition(new, to_state=item.get('state', 'published'))
    new.reindexObject()
    # call recursively for children
    for subitem in item.get('children', []):
        _create_content(subitem, new)

def _constrain(context, allowed_types):
    behavior = ISelectableConstrainTypes(context)
    behavior.setConstrainTypesMode(constrains.ENABLED)
    behavior.setLocallyAllowedTypes(allowed_types)
    behavior.setImmediatelyAddableTypes(allowed_types)

article = {"id":'example','title':u'测试文档','content':u'
这是一个测试文档
'}

def _create_article(article, container):
    id = str(article['id'])

    new = container.get(id, None)
    if not new:
        new = api.content.create(
            type='Document',
            container=container,
            title=article['title'],
            text = RichTextValue(article['content']),
            id=id,
            safe_id=False)
        new.reindexObject()

def import_article(portal):
    "migrate articles to document"

    containers = list(item['id'] for item in STRUCTURE if item['id'] not in ["sqls","help"])

    for con in containers:
        container =  portal[con]
        try:
            _create_article(article,container)
        except:
            continue


import csv

with open('student.csv',encoding = 'utf-8') as f:
    reader = csv.reader(f)
    header_row = next(reader)
    data_rows = list(reader)
header_row.append('Diem TB')

for row in data_rows:
    hs1,hs2,hs3 = int(row[1]),int(row[2]), int(row[3])
    diemtb = round((hs1 + hs2*2 + hs3*3)/6,1)
    row.append(diemtb)

with open('student_bd.csv', 'w',encoding = 'utf-8',newline = '') as f:
    writer = csv.writer(f)
    writer.writerow(header_row)
    writer.writerows(data_rows)


'''
Functions for visualizing data for EDA
'''
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from IPython.display import display, Markdown
from sklearn.preprocessing import MinMaxScaler

# game_data = ['num_comments', 'num_fights', 'attendance', 'len_game', 'corsi_for%',
#              'fenwick_for%', 'faceoff%', 'off_zone_start%', 'pdo', 'total_sog',
#              'total_pp_goals', 'total_goals', 'total_pp_opportunity', 'total_sh_goals',
#              'sog_diff', 'pp_goals_diff', 'goal_diff', 'pp_opportunity_diff', 'sh_goals_diff',
#              'sog_norm_diff', 'pp_goals_norm_diff', 'goal_norm_diff',
#              'pp_opportunity_norm_diff','sh_goals_norm_diff', 'ot/so', 'comeback?', 'win/loss']

game_data = ['num_comments', 'num_fights', 'attendance', 'len_game', 'corsi_for%',
             'fenwick_for%', 'faceoff%', 'off_zone_start%', 'pdo', 'total_sog',
             'total_pp_goals', 'total_goals', 'total_pp_opportunity', 'total_sh_goals',
             'sog_diff', 'pp_goals_diff', 'goal_diff', 'pp_opportunity_diff', 'sh_goals_diff',
             'ot/so', 'comeback?', 'win/loss']

def plot_team_effect_binary_data(data, statistic=None, game_venue='home'):
    '''Plot Means of binary data stats ('win/loss', 'comeback?')
    '''
    if game_venue == 'home':
        home_data = data.groupby(by='home_team').mean()[statistic]
        plt.figure(figsize=(25,10))
        sns.scatterplot(x='home_team', y=statistic, data=home_data.reset_index(drop=False),
                        s=400, marker='s', color='darkblue')
        plt.hlines(y=home_data.mean(), xmin=-1, xmax=30.4, colors='red',
                   label='Population Mean', linewidth=2)

    elif game_venue == 'away':
        away_data = data.groupby(by='away_team').mean()[statistic]
        plt.figure(figsize=(25,10))
        sns.scatterplot(x='away_team', y=statistic, data=away_data.reset_index(drop=False),
                        s=400, marker='s', color='darkblue')
        plt.hlines(y = away_data.mean(), xmin=-1, xmax=30.4, colors='red',
                   label='Population Mean', linewidth=2)

    elif game_venue == 'all':
        home_data = data.groupby(by='home_team').mean()[statistic]
        away_data = data.groupby(by='away_team').mean()[statistic]
        if statistic == 'win/loss':
            away_data = away_data.map(lambda x: 1-x)
        all_data = (home_data + away_data)/2
        all_data.rename_axis('team', inplace=True)
        plt.figure(figsize=(25,10))
        sns.scatterplot(x='team', y=statistic, data=all_data.reset_index(drop=False),
                        s=400, marker='s', color='darkblue')
        plt.hlines(y = all_data.mean(), xmin=-1, xmax=30.4, colors='red',
                   label='Population Mean', linewidth=2)

    title_font = {'size':'22',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20',
                 'weight': 'medium'}
    tick_font = {'size': '16',
                 'weight': 'medium'}
    statistic = statistic.replace('_', ' ').upper()
    plt.title(f'Team Effect on {statistic} in {game_venue.title()} Games', **title_font)
    plt.ylabel(f'{statistic}', **axis_font)
    plt.xlabel('Team', **axis_font)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font);

# def plot_team_effect_binary_data(data, statistic=None, game_venue='home'):
#     '''Plot Means of binary data stats ('win/loss', 'comeback?')
#     '''
#     if game_venue == 'home':
#         home_data = data.groupby(by='home_team').mean()[statistic]
#         plt.figure(figsize=(25,10))
#         sns.scatterplot(x='home_team', y=statistic, data=home_data.reset_index(drop=False),
#                         s=400, marker='s', color='darkblue')
#         plt.hlines(y=home_data.mean(), xmin=-1, xmax=30.4, colors='red',
#                    label='Population Mean', linewidth=2)

#     elif game_venue == 'away':
#         away_data = data.groupby(by='away_team').mean()[statistic]
#         plt.figure(figsize=(25,10))
#         sns.scatterplot(x='away_team', y=statistic, data=away_data.reset_index(drop=False),
#                         s=400, marker='s', color='darkblue')
#         plt.hlines(y = away_data.mean(), xmin=-1, xmax=30.4, colors='red',
#                    label='Population Mean', linewidth=2)

#     title_font = {'size':'22',
#                   'color':'black',
#                   'weight':'medium',
#                   'verticalalignment':'bottom'}
#     axis_font = {'size':'20',
#                  'weight': 'medium'}
#     tick_font = {'size': '16',
#                  'weight': 'medium'}
#     statistic = statistic.replace('_', ' ').upper()
#     plt.title(f'Team Effect on {statistic} in {game_venue.title()} Games', **title_font)
#     plt.ylabel(f'{statistic}', **axis_font)
#     plt.xlabel('Team', **axis_font)
#     plt.xticks(**tick_font)
#     plt.yticks(**tick_font);

def plot_team_effect(data, statistic=None, game_venue='all', kind='boxplot'):
    '''Plot distribution of game statistics in which a given team plays

    statistic = string of game statistic (e.g total_goals, sog_diff). Default = None.

    game_venue = 'home' or 'away' or 'all'. Selects venue to show game stats from (i.e.
    'home' shows game stats for all home games of a given team)

    kind = 'boxplot' or 'violinplot' or 'stripplot'. Selects which type of distribution
    plot to display.
    '''

    if game_venue == 'all':
        teams = sorted(list(data.home_team.unique()))
        team_data = dict()

        for team in teams:
            one_team_data = data.loc[(data.home_team == team) | (data.away_team == team)].reset_index(drop=True)
            team_data[team] = one_team_data[statistic]

        team_data = pd.DataFrame(team_data)

        if kind == 'boxplot':
            plt.figure(figsize=(25,10))
            sns.boxplot(data=team_data)
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                        label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        elif kind == 'violinplot':
            plt.figure(figsize=(25,10))
            sns.violinplot(data=team_data)
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                        label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        elif kind == 'stripplot':
            plt.figure(figsize=(25,10))
            sns.stripplot(data=team_data, jitter=0.35)
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                        label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        else:
            return KeyError('Invalid Plot Kind')

    elif game_venue == 'home' or 'away':

        if kind == 'boxplot':
            plt.figure(figsize=(25,10))
            sns.boxplot(x=f'{game_venue}_team', y=statistic,
                        data=data.sort_values(by=f'{game_venue}_team'))
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                            label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        elif kind == 'violinplot':
            plt.figure(figsize=(25,10))
            sns.violinplot(x=f'{game_venue}_team', y=statistic,
                           data=data.sort_values(by=f'{game_venue}_team'))
            plt.hlines(y=data[statistic].mean(), xmin=-1, xmax=30.4, colors='red',
                            label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        elif kind == 'stripplot':
            plt.figure(figsize=(25,10))
            sns.stripplot(x=f'{game_venue}_team', y=statistic, jitter=0.35,
                         data=data.sort_values(by=f'{game_venue}_team'))
            plt.hlines(y=data[statistic].median(), xmin=-1, xmax=30.4, colors='red',
                            label='Population Median', linewidth=2)
            plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)

        else:
            return KeyError('Invalid Plot Kind')

    else:
        return KeyError('Invalid game_venue')

    title_font = {'size':'22',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20',
                 'weight': 'medium'}
    tick_font = {'size': '16',
                 'weight': 'medium'}
    sns.despine(offset=0, trim=True)
    statistic = statistic.replace('_', ' ').upper()
    plt.title(f'Team Effect on {statistic} in {game_venue.title()} Games', **title_font)
    plt.ylabel(f'{statistic}', **axis_font, labelpad=5)
    plt.xlabel('Team', **axis_font, labelpad=5)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font);

def plot_team_specific_stats(data, statistic=None, against=False):
    '''Plot distribution of team-specific statistics

    statistic = string of statistic that is available for both home and away (e.g corsi,
    goals, sog, etc). Default = None.

    against = Boolean. False will show distribution of statistic for a given team, True will
    show distribution of statistic against a given team. Default = False.
    '''
    teams = list(data.home_team.unique())
    team_data = dict()

    for team in teams:
        if against == False:
            home_statistic = data.loc[(data.home_team == team)]['home_' + statistic]
            away_statistic = data.loc[(data.away_team == team)]['away_' + statistic]
        elif against == True:
            home_statistic = data.loc[(data.home_team == team)]['away_' + statistic]
            away_statistic = data.loc[(data.away_team == team)]['home_' + statistic]

        statistic_data = pd.concat([home_statistic, away_statistic], ignore_index=True)
        team_data[team] = statistic_data

    team_data = pd.DataFrame(team_data)
    plt.figure(figsize=(25,10))
    sns.boxplot(data=team_data)
    population_stat_data = (list(data['home_' + statistic]) +
                            (list(data['away_' + statistic])))
    plt.hlines(y=np.median(population_stat_data), xmin=-1, xmax=30.4,
               colors='red', label='Population Median', linewidth=2)
    plt.legend(loc=(0,0.95), fontsize=16, edgecolor=None)
    title_font = {'size':'22',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20', 'weight': 'medium'}
    tick_font = {'size': '16', 'weight': 'medium'}
    sns.despine(offset=0, trim=True)

    if against == False:
        statistic = statistic + '_for'
    elif against == True:
        statistic = statistic + '_against'

    statistic = statistic.replace('_', ' ').upper()
    plt.title(f'Team Effect on {statistic} in Games', **title_font)
    plt.ylabel(f'{statistic}', **axis_font, labelpad=5)
    plt.xlabel('Team', **axis_font, labelpad=5)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font);

def plot_numeric_features_dist(data, kind='boxplot', size=(12,8), normalize=True):
    '''
    Plot distributions of numeric features within a DataFrame

    size = tuple with desired size of output figure (width, height)

    normalize = Boolean (Default = True). If True, numerical data is normalized using
    sklearn.preprocessing.MinMaxScaler(). If False, numerical data is not altered.
    '''
    columns = []
    for column in data.columns:
        if (data[column].dtype != 'O'):
            columns.append(column)
    if normalize == True:
        scaler = MinMaxScaler()
        scaled_data = scaler.fit_transform(data[columns])
        data = pd.DataFrame(scaled_data, columns=columns)
    if kind == 'boxplot':
        plt.figure(figsize=(size))
        sns.boxplot(data=data[columns])
    elif kind == 'violinplot':
        plt.figure(figsize=(size))
        sns.violinplot(data=data[columns], scale='count', cut=0)
    elif kind == 'stripplot':
        plt.figure(figsize=(size))
        sns.stripplot(data=data[columns], jitter=0.25)
    title_font = {'size':'18',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    tick_font = {'size': '12',
                 'weight': 'medium'}
    if normalize == True:
        plt.title(f'Normalized Distribution of Numeric Features', **title_font)
    elif normalize == False:
        plt.title(f'Distribution of Numeric Features', **title_font)
    formatted_x_ticks = [x.replace('_', ' ').replace(' polarity', '').title() for x in columns]
    plt.yticks(**tick_font)
    plt.xticks(ticks=plt.xticks()[0], labels=formatted_x_ticks, rotation=60, ha='right', **tick_font);

def plot_game_binary_data(data, size=(16, 5)):
    '''
    Plot distributions of binary/ordinal game data
    '''

    title_font = {'size':'16',
              'color':'black',
              'weight':'medium',
              'verticalalignment':'bottom'}
    axis_font = {'size':'14', 'weight': 'medium'}
    tick_font = {'size': '16', 'weight': 'medium'}

    fig, axs = plt.subplots(1, len(data.columns), figsize=size)
    for i, column in enumerate(data.columns):
        axs[i].hist(data[column])
        column = column.upper()
        axs[i].set_title(column, **title_font)
    axs[0].set_ylabel('Frequency', **axis_font);

def plot_team_effect_comment_fandom(comment_data, size=(12,6)):
    '''
    Plots percentage of comments from a team fanbase that are in game threads that their respective team played in

    comment_data = output of clean_data.extract_comment_data()
    '''
    nhl_abbreviations = ['CBJ', 'TBL', 'MTL', 'FLA', 'NSH', 'SJS', 'LAK', 'BOS', 'COL', 'OTT', 'DAL', 'CGY', 'MIN',
                    'NYI', 'NYR', 'BUF', 'WSH', 'PIT', 'WPG', 'VAN', 'PHI', 'CHI', 'CAR', 'ANA', 'EDM', 'STL',
                    'DET', 'TOR', 'NJD', 'ARI', 'VGK']

    title_font = {'size':'20',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'16', 'weight': 'medium'}
    tick_font = {'size': '14', 'weight': 'medium'}

    plt.figure(figsize=size)
    comment_data.groupby('flair').mean().loc[nhl_abbreviations]['fan'].plot(kind='bar')
    plt.hlines(y=np.mean(comment_data.groupby('flair').mean().loc[nhl_abbreviations]['fan']), xmin=0, xmax=31, color='red')
    plt.title(label='Percentage of Fan Comments by Team', **title_font)
    plt.xlabel('Team', **axis_font)
    plt.ylabel('Percetage of Fan Comments', **axis_font)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font);

def plot_scoring_dist(comment_data, scoring=None, size=(10, 6)):
    '''
    Plot distributions of binary/ordinal game data
    '''
    perc_neutral = np.round(len(comment_data.loc[comment_data[scoring] == 0])/len(comment_data),
                            decimals=3)
    title_font = {'size':'16',
              'color':'black',
              'weight':'medium',
              'verticalalignment':'bottom'}
    axis_font = {'size':'14', 'weight': 'medium'}
    tick_font = {'size': '12', 'weight': 'medium'}

    plt.figure(figsize=size)
    comment_data[scoring].plot(kind='hist', bins=100)
    plt.ylabel('Frequency', **axis_font)
    scoring = scoring.replace('_', ' ').title()
    plt.xlabel(f'{scoring}', **axis_font)
    plt.xticks(**tick_font)
    plt.yticks(**tick_font)
    plt.show();

    print(f'Percentage of neutral comments: {perc_neutral}')

def plot_nlp_heatmap(thread_data, size=(14,17)):
    '''
    Plot heatmap showing correlation between sentiment scores and game features
    '''
    nlp_data = ['fan%', 'vader_comment',
       'socialsent_comment', 'vader_context',
       'socialsent_context', 'positive_context', 'non-fan_context',
       'no_ref_context']

    corr_data = thread_data[game_data + nlp_data].corr().applymap(lambda x: np.round(x, 2))
    corr_data = corr_data.drop(index=nlp_data)[nlp_data]

    color_bar = {
        'shrink': 0.5,
        'pad': 0.01
    }

    title_font = {'size':'20',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20',
                 'weight': 'medium'}
    tick_font = {'size': '12',
                 'weight': 'medium'}

    plt.figure(figsize=size)
    sns.heatmap(corr_data, annot=True, linewidths=0.1,
                square=False, cbar=True, cbar_kws=color_bar,
               vmin=-1, vmax=1, cmap="RdBu_r");
    plt.xticks(rotation=30, ha='right', **tick_font)
    plt.yticks(rotation=0, ha='right', **tick_font)

    plt.title('Correlation of Game Features with Sentiment Scores', **title_font);

def plot_game_feature_heatmap(total_data):
    '''
    Plot showing correlation between game features
    '''
    corr_data = total_data[game_data].corr().applymap(lambda x: np.round(x, 2))

    mask = np.zeros_like(corr_data)
    mask[np.triu_indices_from(mask)] = True

    color_bar = {
        'shrink': 0.5,
        'pad': 0.01
    }

    title_font = {'size':'20',
                  'color':'black',
                  'weight':'medium',
                  'verticalalignment':'bottom'}
    axis_font = {'size':'20',
                 'weight': 'medium'}
    tick_font = {'size': '12',
                 'weight': 'medium'}

    plt.figure(figsize=(17,17))
    sns.heatmap(corr_data, annot=True, linewidths=0.1,
                square=False, cbar=True, cbar_kws=color_bar,
               vmin=-1, vmax=1, cmap="RdBu_r", mask=mask);
    plt.xticks(rotation=90, ha='right', **tick_font)
    plt.yticks(rotation=0, ha='right', **tick_font)

    plt.title('Correlation of Game Features', **title_font);


from django.urls import path
from .views import (
    PerkDetail,
    Perks,
    Experiences,
    ExperienceDetail,
    ExperiencePerk,
    ExperienceBooking,
    ExperienceBookingAll,
    ExperienceBookingDetail,
)

urlpatterns = [
    path("", Experiences.as_view()),
    path("<int:pk>/perks", ExperiencePerk.as_view()),
    path("<int:pk>", ExperienceDetail.as_view()),
    path("<int:pk>/bookings/all", ExperienceBookingAll.as_view()),
    path("<int:pk>/bookings/", ExperienceBooking.as_view()),
    path("<int:pk>/bookings/<int:booking_pk>", ExperienceBookingDetail.as_view()),
    path("perks/", Perks.as_view()),
    path("perks/<int:pk>", PerkDetail.as_view()),
]


class Solution(object):
  def nthUglyNumber(self, n):
    """
    :type n: int
    :rtype: int
    """
    dp = [0] * (n + 1)
    dp[1] = 1
    i2 = i3 = i5 = 1
    for i in range(2, n + 1):
      dp[i] = min(dp[i2] * 2, dp[i3] * 3, dp[i5] * 5)
      if dp[i] == dp[i2] * 2:
        i2 += 1
      if dp[i] == dp[i3] * 3:
        i3 += 1
      if dp[i] == dp[i5] * 5:
        i5 += 1
    return dp[-1]


import numpy as np
# import minpy.numpy as np
import bpnn
import data_utils
from plot import *
def act_type():

    # 加载 npy 文件
    print("Doing: load npy files")
    X_train, X_test, y_train, y_test = data_utils.load_npy("../")

    X_train = X_train[0:10000]
    y_train = y_train[0:10000]

    # 将单幅图片转成 3072 维的向量
    print("Doing: image data reshape")
    X_train = np.reshape(X_train, (X_train.shape[0], -1))
    X_test = np.reshape(X_test, (X_test.shape[0], -1))

    models = [
                [[3072,200,100,50,10],[bpnn.relu,bpnn.relu,bpnn.relu,bpnn.relu,bpnn.relu]],
                [[3072,200,100,50,10],[bpnn.sigmoid, bpnn.sigmoid, bpnn.sigmoid, bpnn.sigmoid, bpnn.sigmoid]],
                [[3072,200,100,50,10],[bpnn.tanh,bpnn.tanh,bpnn.tanh,bpnn.tanh,bpnn.tanh]],
                [[3072,200,100,50,10],[bpnn.unact,bpnn.unact,bpnn.unact,bpnn.unact,bpnn.unact]]
            ]
    lrs = [1e-2]
    act = ["relu", "sigmoid", "tanh", "unact"]
    for i, m in enumerate(models):
        for lr in lrs:
            classifier = bpnn.BPNN(m, 'xavier')
            optimizer = bpnn.Optimizer(classifier)

            optimizer.lr = lr
            optimizer.lr_decay = bpnn.Learning_rate_decay.exp
            optimizer.lr_k = 0
            optimizer.momentum_type = bpnn.Momentum.Adagrad
            optimizer.mu = 0.8
            optimizer.reg_type = bpnn.Regularization.L2
            optimizer.reg = 0.00075

            classifier.useOpt(optimizer)

            print("Doing: train net")
            train_loss_list, train_acc_list = classifier.train(X_train, y_train, epoch=50, batch_size=64)
            print(train_loss_list[-1], train_acc_list[-1])
            draw_plot(range(len(train_loss_list)), train_loss_list, "loss_" + act[i], "epoch", "loss", "loss_" + act[i])
            draw_plot(range(len(train_acc_list)), train_acc_list, "train_acc_" + act[i], "epoch", "train_acc", "train_acc_" + act[i])
            test_acc = classifier.evaluate(X_test, y_test)
            print("test accuracy is %f" % (test_acc))

    #print(train_loss_list)
    #print("Doing: test net")

if __name__ == "__main__":
    act_type()
    '''
    print("Doing: load image data")
    X_train, y_train, X_test, y_test = data_utils.load_CIFAR10('../cifar-10-batches-py')

    # 图像左右翻转增加数据集
    print("Doing: image enhance")
    data_utils.create_image_enhance_npy(X_train, X_test, y_train, y_test, "../")
    '''


'''
Scrapes hockey reference for game data and team-specific game data
    -Use scrape_game_data() for game data
    -Use scrape_team_data() for team-specific game data
'''

import pandas as pd
import requests
from bs4 import BeautifulSoup
import time
from tqdm import tqdm_notebook

team_name = ['San Jose Sharks',
             'Toronto Maple Leafs',
             'Vancouver Canucks',
             'Washington Capitals',
             'Buffalo Sabres',
             'Carolina Hurricanes',
             'Colorado Avalanche',
             'Dallas Stars',
             'Detroit Red Wings',
             'New York Rangers',
             'Ottawa Senators',
             'Pittsburgh Penguins',
             'St. Louis Blues',
             'Vegas Golden Knights',
             'Columbus Blue Jackets',
             'Los Angeles Kings',
             'Arizona Coyotes',
             'Calgary Flames',
             'Minnesota Wild',
             'New Jersey Devils',
             'New York Islanders',
             'Tampa Bay Lightning',
             'Chicago Blackhawks',
             'Anaheim Ducks',
             'Boston Bruins',
             'Nashville Predators',
             'Philadelphia Flyers',
             'Winnipeg Jets',
             'Florida Panthers',
             'Montreal Canadiens',
             'Edmonton Oilers']

team_code = ['SJS',
            'TOR',
            'VAN',
            'WSH',
            'BUF',
            'CAR',
            'COL',
            'DAL',
            'DET',
            'NYR',
            'OTT',
            'PIT',
            'STL',
            'VEG',
            'CBJ',
            'LAK',
            'ARI',
            'CGY',
            'MIN',
            'NJD',
            'NYI',
            'TBL',
            'CHI',
            'ANA',
            'BOS',
            'NSH',
            'PHI',
            'WPG',
            'FLA',
            'MTL',
            'EDM']

name_to_code = dict(zip(team_name, team_code))

def scrape_team_data(years):
    '''
    Scrape all team stats for specified years from https://www.hockey-reference.com/

    Inputs:

    years -> An iterable of years to scrape

    Outputs:

    Saves a pickle for each year containing:

        -list of tuples
        -tuple[0] is team 3-letter code
        -tuple[1] is a pandas dataframe with team stats
    '''
    for year in years:
        teams = ['ANA',
                'ARI',
                'BOS',
                'BUF',
                'CAR',
                'CGY',
                'CHI',
                'CBJ',
                'COL',
                'DAL',
                'DET',
                'EDM',
                'FLA',
                'LAK',
                'MIN',
                'MTL',
                'NSH',
                'NJD',
                'NYI',
                'NYR',
                'OTT',
                'PHI',
                'ARI',
                'PIT',
                'SJS',
                'STL',
                'TBL',
                'VAN',
                'TOR',
                'VEG',
                'WPG',
                'WSH']
        # Vegas Golden Knight Entered NHL in 2017-2018 Season
        if year < 2017:
            teams.remove('VEG')

        team_data = []
        for team in tqdm_notebook(teams):
            url = f'https://www.hockey-reference.com/teams/{team}/{year}_gamelog.html'
            headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'}
            resp2 = requests.get(url=url, headers=headers)
            soup = BeautifulSoup(resp2.text, features='lxml')
            table = soup.find('table')
            table_body = table.find('tbody')
            rows = table_body.find_all('tr')
            data= []
            for row in rows:
                cols = row.find_all('td')
                games_played = row.find('th')
                cols = [ele.text.strip() for ele in cols]
                cols.append(games_played.text)
                data.append(cols)
            team_game_data_df = pd.DataFrame(data)
            team_game_data_df.columns = ['date',
                                    '?',
                                    'opponent',
                                    'goals_for',
                                    'goals_against',
                                    'win/loss',
                                    'ot/so',
                                    '?',
                                    'team_sog',
                                    'team_pim',
                                    'team_pp_goal',
                                    'team_pp_oppurtunities',
                                    'team_shorthanded_goals',
                                    '?',
                                    'opponent_sog',
                                    'opponent_pim',
                                    'opponent_pp_goal',
                                    'opponent_pp_oppurtunities',
                                    'opponent_shorthanded_goals',
                                    '?',
                                    'corsi_for',
                                    'corsi_against',
                                    'corsi_for%',
                                    'fenwick_for',
                                    'fenwick_against',
                                    'fenwick_for%',
                                    'faceoff_wins',
                                    'faceoff_loses',
                                    'faceoff%',
                                    'off_zone_start%',
                                    'pdo',
                                    'games_played'
                                    ]
            team_game_data_df.drop(columns=['?'], inplace=True)
            team_game_data_df.dropna(inplace=True)
            team_game_data_df['date'] = team_game_data_df['date'].map(lambda x: pd.to_datetime(x))
            team_game_data_df['opponent'] = team_game_data_df['opponent'].map(lambda x: name_to_code[x])
            team_data.append((team, team_game_data_df))
            time.sleep(2)
        pd.to_pickle(team_data, f'./data/team_game_data_{year}')

def scrape_game_data(years):
    '''
    Scrape all game stats for specified years from https://www.hockey-reference.com/

    Inputs:

    years -> An iterable of years to scrape

    Outputs:

    Pickles a dataframe for each year
    '''
    for year in years:
        url = f'https://www.hockey-reference.com/leagues/NHL_{year}_games.html'
        headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'}
        resp = requests.get(url=url, headers=headers)
        soup = BeautifulSoup(resp.text, features='lxml')
        table = soup.find('table')
        table_body = table.find('tbody')
        rows = table_body.find_all('tr')

        data= []
        for row in rows:
            cols = row.find_all('td')
            date = row.find('th')
            cols = [ele.text.strip() for ele in cols]
            cols.append(date.text)
            data.append(cols)

        game_data_df = pd.DataFrame(data)
        game_data_df.columns = ['visitor', 'visitor_goals', 'home',
                                'home_goals', 'ot/so', 'attendance',
                                'len_game', 'notes', 'date']
        game_data_df.drop(columns=['notes'], inplace=True)
        game_data_df['visitor'] = game_data_df['visitor'].map(lambda x: name_to_code[x])
        game_data_df['home'] = game_data_df['home'].map(lambda x: name_to_code[x])
        game_data_df['date'] = game_data_df['date'].map(lambda x: pd.to_datetime(x))

        pd.to_pickle(game_data_df, f'./data/all_game_data_{year}')


#!/usr/bin/env python

import datetime
import json
import logging
import os
import pdb
from sqlalchemy import and_
import time
import uuid

from vib.vwf.VWorker import VWorker

import vib.config.AppConfig
config = vib.config.AppConfig.AppConfig( 'viblio' ).config()

import vib.utils.s3 as s3
import vib.db.orm
from vib.db.models import *

import vib.vwf.Transcode.transcode_utils as tutils

log = logging.getLogger( __name__ )

class Transcode( VWorker ):
    # This line controls how we interact with SWF, and changes here
    # must be made in coordination with VPWorkflow.py
    task_name = 'Transcode'

    def run_task( self, options ):
        '''Transcode a video.  Input options are:
        { media_uuid, user_uuid, input_file : { s3_bucket, s3_key },
          metadata_uri,
          original_uuid,
          outputs : [ { output_file : { s3_bucket, s3_key},
                      format : "mp4",
                      max_video_bitrate: 1500,
                      audio_bitrate : 160,
                      scale: "640:-1", # Passed to ffmpeg -vf scale argument, overrides size
                      size: "640x360", # Currently ignored
                      asset_type: "main",
                      thumbnails : [ {
                        times : [0.5], size: "320x240", label: "poster",
                        format : "png",
                        output_file: { s3_bucket, s3_key } } ]
                     } ] }
                     # If multiple output_files are present, only one need have thumbnails.
        '''
        try:
            media_uuid = options['media_uuid']
            user_uuid = options['user_uuid']
            input_file = options['input_file']
            outputs = options['outputs']

            log.info( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Starting transcode for user %s media %s" % ( user_uuid, media_uuid )
                    } ) )

            # Download to our local disk.
            original_file = config.transcode_dir + "/"  + media_uuid
            log.debug( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Downloading original file from s3 for user %s media %s" % ( user_uuid, media_uuid )
                    } ) )

            s3.download_file( original_file, input_file['s3_bucket'], input_file['s3_key'] )

            # There seems to be a race condition where somehow the OS
            # doesn't present the file to the next command sometimes -
            # give things a few seconds to catch up.
            time.sleep( 5 )

            log.debug( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Generating exif for original file for user %s media %s" % ( user_uuid, media_uuid )
                    } ) )
            exif = tutils.get_exif( media_uuid, original_file, user_uuid )

            try:
                log.debug( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'message' : "Running qt-faststart for user %s media %s" % ( user_uuid, media_uuid )
                    } ) )
                tutils.move_atom( media_uuid, original_file, user_uuid )
            except Exception as e:
                log.warning( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'message' : "Exception thrown while running qt-faststart for user %s media %s : %s" % ( user_uuid, media_uuid, e ) } ) )
                # Explicitly do nothing upon error in this case.

            # Process files into S3
                log.debug( json.dumps( { 'media_uuid' : media_uuid,
                                         'user_uuid' : user_uuid,
                                         'message' : "Transcoding and storing the result in S3 for user %s media %s : " % ( user_uuid, media_uuid ) } ) )

            outputs = tutils.transcode_and_store( media_uuid, original_file, outputs, exif, options.get( 'try_photos', False ), user_uuid )

            orm = vib.db.orm.get_session()
            orm.commit()

            # Get the media object to which all our subordinate files relate.
            media = orm.query( Media ).filter( Media.uuid == media_uuid ).first()
            if media is None:
                log.error( json.dumps( { 'media_uuid' : media_uuid,
                                         'user_uuid' : user_uuid,
                                         'message' : 'Failed to locate database record for media %s, perhaps it has been deleted?' % ( media_uuid ) } ) )
                # Terminate execution permanently, if there is no
                # database record for this we can't possibly proceed.
                return { 'ACTIVITY_ERROR' : True, 'retry' : False }

            if options.get( 'viblio_added_content_type', '' ) != '':
                # This is a viblio generated video, set it's location to Palo Alto
                media.lat = 37.442174
                media.lng = -122.143199
            else:
                media.lat = exif['lat']
                media.lng = exif['lng']

            media.status = 'visible'

            mwfs = MediaWorkflowStages( workflow_stage = self.task_name + 'Complete' )
            media.media_workflow_stages.append( mwfs )

            # Calculate the recording date for video versions.
            if options.get( 'viblio_added_content_type', '' ) != '':
                # This is a viblio generated video, the recording date should be now.
                recording_date = datetime.datetime.now()
            else:
                recording_date = datetime.datetime.utcfromtimestamp( 0 )

            if exif['create_date'] and exif['create_date'] != '' and exif['create_date'] != '0000:00:00 00:00:00':
                try:
                    recording_date = datetime.datetime.strptime( exif['create_date'], '%Y:%m:%d %H:%M:%S' )
                except Exception as e:
                    recording_date = exif['create_date']
            log.debug( 'Setting recording date to ' + str( recording_date ) )
            log.debug( 'Exif data for create was ' + exif['create_date'] )
            media.recording_date = recording_date

            original = orm.query( MediaAssets ).filter( MediaAssets.uuid == options['original_uuid'] ).first()
            if original is None:
                log.error( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'message' : 'Failed to locate database record for original media asset %s, perhaps it has been deleted?' % ( media_uuid ) } ) )
                # Terminate execution permanently, if there is no
                # database record for this we can't possibly proceed.
                return { 'ACTIVITY_ERROR' : True, 'retry' : False }

            original.mimetype = 'video/%s' % exif.get( 'format', 'mp4' )
            original.width = exif.get( 'width', None )
            original.height = exif.get( 'height', None )

            # We will return to the next stage the key/bucket of the
            # "main" asset_type.
            return_bucket = None
            return_key = None

            for output in outputs:
                output_uuid = str( uuid.uuid4() )

                output_exif = tutils.get_exif( media_uuid, output['output_file_fs'], user_uuid )

                log.info( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'asset_type' : output['asset_type'],
                            'output_uuid' : output_uuid,
                            'message' : "Creating video database row of uuid %s for user %s, media %s of asset_type %s and uri %s" % ( output_uuid, user_uuid, media_uuid, output['asset_type'], output['output_file']['s3_key'] )
                    } ) )

                if output['asset_type'] == 'main':
                    return_bucket = output['output_file']['s3_bucket']
                    return_key = output['output_file']['s3_key']

                video_asset = MediaAssets(
                    uuid         = output_uuid,
                    asset_type   = output['asset_type'],
                    mimetype     = 'video/%s' % exif.get( 'format', 'mp4' ),
                    metadata_uri = options['metadata_uri'],
                    bytes        = os.path.getsize( output['output_file_fs'] ),
                    uri          = output['output_file']['s3_key'],
                    location     = 'us',
                    view_count   = 0,
                    width        = output_exif.get( 'width', None ),
                    height       = output_exif.get( 'height', None ),
                    duration     = output_exif['duration'] )
                media.assets.append( video_asset )

                if output['asset_type'] == 'main' and recording_date != datetime.datetime.utcfromtimestamp( 0 ):
                    try:
                        # Add a Month Year tag to the main media asset.
                        date_string = recording_date.strftime( "%B %Y" )
                        date_tag = MediaAssetFeatures(
                            feature_type = 'activity',
                            coordinates = date_string
                        )
                        video_asset.media_asset_features.append( date_tag )
                    except Exception as e:
                        log.warning( json.dumps( { 'media_uuid' : media_uuid,
                                                   'user_uuid' : user_uuid,
                                                   'message' : 'Failed to add date tag for video, error was: %s' % ( e ) } ) )

                video_poster = None

                for thumbnail in output.get( 'thumbnails', [] ):
                    thumbnail_uuid = str( uuid.uuid4() )

                    log.info( json.dumps( {
                                'media_uuid' : media_uuid,
                                'user_uuid' : user_uuid,
                                'asset_type' : thumbnail['label'],
                                'output_uuid' : thumbnail_uuid,
                                'message' : "Creating image database row of uuid %s for user %s, media %s of asset_type %s and uri %s" % ( thumbnail_uuid, user_uuid, media_uuid, thumbnail['label'], thumbnail['output_file']['s3_key'] )
                                } ) )

                    thumbnail_size = thumbnail.get( 'size', "320x240" )
                    thumbnail_x, thumbnail_y = thumbnail_size.split( 'x' )
                    thumbnail_asset = MediaAssets( uuid       = thumbnail_uuid,
                                                   asset_type = thumbnail['label'],
                                                   mimetype   = 'image/%s' % thumbnail.get( 'format', 'png' ),
                                                   bytes      = os.path.getsize( thumbnail['output_file_fs'] ),
                                                   width      = int( thumbnail_x ),
                                                   height     = int( thumbnail_y ),
                                                   uri        = thumbnail['output_file']['s3_key'],
                                                   location   = 'us',
                                                   view_count = 0 )
                    media.assets.append( thumbnail_asset )

                    if thumbnail['label'] == 'poster':
                        video_poster = thumbnail_asset

            # Determine if we need to create and or add to the special Video summary album.
            if options.get( 'viblio_added_content_type', '' ) == config.viblio_summary_video_type:
                viblio_summary_album = orm.query( Media ).filter( and_( Media.user_id == media.user_id, Media.is_viblio_created == True, Media.title == config.viblio_summary_album_name ) )[:]

                if len( viblio_summary_album ) == 0:
                    viblio_summary_album = Media( user_id = media.user_id,
                                                  uuid = str( uuid.uuid4() ),
                                                  media_type = 'original',
                                                  is_album = True,
                                                  title = config.viblio_summary_album_name,
                                                  is_viblio_created = True )
                    orm.add( viblio_summary_album )

                    media_album_row = MediaAlbums()
                    orm.add( media_album_row )
                    media.media_albums_media.append( media_album_row )
                    viblio_summary_album.media_albums.append( media_album_row )

                    album_poster = MediaAssets( user_id = media.user_id,
                                                uuid = str( uuid.uuid4() ),
                                                asset_type = 'poster',
                                                mimetype = video_poster.mimetype,
                                                location = video_poster.location,
                                                uri = video_poster.uri,
                                                width = video_poster.width,
                                                height = video_poster.height )
                    viblio_summary_album.assets.append( album_poster )

                elif len( viblio_summary_album ) == 1:
                    media_album_row = MediaAlbums()
                    orm.add( media_album_row )
                    media.media_albums_media.append( media_album_row )
                    viblio_summary_album[0].media_albums.append( media_album_row )
                else:
                    raise Exception( "ERROR: Found multiple %s albums for user: %s " % ( config.viblio_summary_album_name, media.user_id ) )

            # Determine if we need to create and or add to the special Try Photo Finder album.
            if options.get( 'try_photos', False ):
                try_photos_album = orm.query( Media ).filter( and_( Media.user_id == media.user_id, Media.is_viblio_created == True, Media.title == config.try_photos_album_name ) )[:]

                if len( try_photos_album ) == 0:
                    try_photos_album = Media( user_id = media.user_id,
                                              uuid = str( uuid.uuid4() ),
                                              media_type = 'original',
                                              is_album = True,
                                              title = config.try_photos_album_name,
                                              is_viblio_created = True )
                    orm.add( try_photos_album )

                    media_album_row = MediaAlbums()
                    orm.add( media_album_row )
                    media.media_albums_media.append( media_album_row )
                    try_photos_album.media_albums.append( media_album_row )

                    album_poster = MediaAssets( user_id = media.user_id,
                                                uuid = str( uuid.uuid4() ),
                                                asset_type = 'poster',
                                                mimetype = video_poster.mimetype,
                                                location = video_poster.location,
                                                uri = video_poster.uri,
                                                width = video_poster.width,
                                                height = video_poster.height )
                    try_photos_album.assets.append( album_poster )

                elif len( try_photos_album ) == 1:
                    media_album_row = MediaAlbums()
                    orm.add( media_album_row )
                    media.media_albums_media.append( media_album_row )
                    try_photos_album[0].media_albums.append( media_album_row )
                else:
                    raise Exception( "ERROR: Found multiple %s albums for user: %s " % ( config.try_photos_album_name, media.user_id ) )

            log.info( json.dumps( { 'media_uuid' : media_uuid,
                                    'user_uuid' : user_uuid,
                                    'message' : "Committing rows to database for user %s, media %s" % ( user_uuid, media_uuid ) } ) )
            orm.commit()

            self.cleanup_files( media_uuid, user_uuid, original_file, outputs )

            return_value = {
                'media_uuid' : media_uuid,
                'user_uuid' : user_uuid,
                'output_file' : {
                    's3_bucket' : return_bucket,
                    's3_key' : return_key
                    }
                }

            return return_value

        except Exception as e:
            log.error( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Exception while transcoding video for user %s, media %s, error was: %s" % ( user_uuid, media_uuid, e )
                        } ) )

            self.cleanup_files( media_uuid, user_uuid, original_file, outputs )
            raise

    def cleanup_files( self, media_uuid, user_uuid, original_file, outputs ):
        try:
            # Delete the original file
            if os.path.exists( original_file ):
                log.debug( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'message' : "Deleting temporary file %s for user %s, media %s" % ( original_file, user_uuid, media_uuid )
                            } ) )
                os.remove( original_file )

                exif_file = os.path.splitext( original_file )[0] + '_exif.json'

                if os.path.exists( exif_file ):
                    log.debug( json.dumps( {
                                'media_uuid' : media_uuid,
                                'user_uuid' : user_uuid,
                                'message' : "Deleting temporary file %s for user %s, media %s" % ( exif_file, user_uuid, media_uuid )
                                } ) )
                    os.remove( exif_file )

            # Delete file files created and stored in the outputs data
            # structure.
            for output in outputs:
                if 'output_file_fs' in output:
                    if os.path.exists( output['output_file_fs'] ):
                        log.debug( json.dumps( {
                                    'media_uuid' : media_uuid,
                                    'user_uuid' : user_uuid,
                                    'message' : "Deleting temporary file %s for user %s, media %s" % ( output['output_file_fs'], user_uuid, media_uuid )
                                } ) )
                        os.remove( output['output_file_fs'] )

                        exif_file = os.path.splitext( output['output_file_fs'] )[0] + '_exif.json'

                        if os.path.exists( exif_file ):
                            log.debug( json.dumps( {
                                        'media_uuid' : media_uuid,
                                        'user_uuid' : user_uuid,
                                        'message' : "Deleting temporary file %s for user %s, media %s" % ( output['output_file_fs'], user_uuid, media_uuid )
                                        } ) )
                            os.remove( exif_file )

                for thumbnail in output.get( 'thumbnails', [] ):
                    if 'output_file_fs' in thumbnail:
                        if os.path.exists( thumbnail['output_file_fs'] ):
                            log.debug( json.dumps( {
                                        'media_uuid' : media_uuid,
                                        'user_uuid' : user_uuid,
                                        'message' : "Deleting temporary file %s for user %s, media %s" % ( thumbnail['output_file_fs'], user_uuid, media_uuid )
                                        } ) )
                            os.remove( thumbnail['output_file_fs'] )

        except Exception as e:
            log.error( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Exception while cleaning up temporary files for user %s, media %s, error was: %s" % ( user_uuid, media_uuid, e )
                        } ) )
            raise


import pandas as pd
from vpy.analysis import Analysis
from vpy.sim import Sim
from vpy.standard.frs5.uncert import Uncert
from vpy.standard.frs5.cal import Cal

def main():

    sim = Sim("frs5")
    doc = sim.build()

    res = Analysis(doc)
    unc = Uncert(doc)
    cal = Cal(doc)

    cal.pressure_cal(res)
    unc.total(res)
    p = res.pick("Pressure", "frs5", "mbar")
    u = res.pick("Uncertainty", "frs5_total_rel", "1")

    p_f = ['{:.1e}'.format(i) for i in p]
    u_f = ['{:.1e}'.format(i) for i in u]
    df = pd.DataFrame({"$p_{frs5}$ in mbar":p_f, "$u(k=1)$ (relative)":u_f})

    print(df.to_latex())
if __name__ == "__main__":
    main()


import os

default_locale = 'EN'
strings = {
    'string_a': {'EN': 'EN_AAAAAA', 'IT': 'IT_AAAAA'},
    'string_b': {'EN': 'EN_BBBBBB'},
    'tt2': {'EN': 'EN_BBBBBB'},
}

num_buckets = 16

FNV32_BASE = 0x811c9dc5
FNV32_PRIME = 0x01000193

# File doesn't exist or out of date
def needs_regen(path, compare_ts):
    return not os.path.isfile(path) \
        or os.path.getmtime(path) < compare_ts

def max_first(arr):
    m = -1
    obj = None

    for (l, o) in arr:
        if l > m:
            m = l
            obj = o

    return obj

extracted_langs = list(
    max_first(
        ((len(
            strings[key]), dict.keys(
                strings[key])) for key in strings)))
num_langs = len(extracted_langs)

strings_heap = b''
offset = 0

buckets = [[] for x in range(0, num_buckets)]

def hash_str(string):
    h = FNV32_BASE
    for c in string.encode('ASCII'):
        h = ((h * FNV32_PRIME) & 0xFFFFFFFF) ^ c

    return h

def get_offset(string):
    global strings_heap
    global offset

    enc = string.encode('ASCII')
    len_enc = len(enc)

    strings_heap += enc + b'\x00'
    offset += len_enc + 1

    return offset - len_enc - 1

class HeapObj:
    def __init__(self, key, values):
        self.key = get_offset(key)

        self.arr = []
        for lang in extracted_langs:
            val = values.get(lang)
            self.arr.append(get_offset(val) if val else -1)

    def to_c(self, name, prev):
        str_struct = '{{ {} }}'.format(
            ','.join(['0' if x == -1
                      else 'str_heap+{}'.format(x) for x in self.arr])
        )

        return 'static const locale_map_node_t node_{} = \
{{ .key = {}, .value = {}, .next = {} }};'.format(
            name, 'str_heap+{}'.format(self.key), str_struct,
            '&node_{}'.format(prev) if prev else '0')

for key in strings:
    if not strings[key].get(default_locale):
        raise ValueError(
            'Missing value for default locale in string `{}`'.format(key))

    obj = HeapObj(key, strings[key])
    b = hash_str(key) % num_buckets
    buckets[b].append(obj)

out_h_path = 'autogen_lang.h'
out_c_path = 'autogen_lang.c'
this_mtime = os.path.getmtime(__file__)

# For consistency, let's say if either file needs regenerating, we
# regenerate both - just to ensure we always have a matching pair.
if needs_regen(out_h_path, this_mtime) or needs_regen(out_c_path, this_mtime):
    with open(out_c_path, 'w+') as f:
        f.write('#include "jlocale.h"\n')

        str_heap_to_string = str(strings_heap)
        str_heap_to_string = str_heap_to_string.replace('\\x00', '\\0')
        f.write('static const char str_heap[] = "{}";\n'.format(
            str_heap_to_string[2:]))

        for b in range(0, num_buckets):
            for i, s in enumerate(buckets[b]):
                prev = '{}_{}'.format(b, i - 1) if i else None
                f.write(s.to_c('{}_{}'.format(b, i), prev))
                f.write('\n')

        f.write('const locale_map_t default_map = {')
        f.write('.buckets = {')
        for b in range(0, num_buckets):
            len_b = len(buckets[b])
            f.write('&node_{}_{},'.format(b, len_b - 1) if len_b else '0,')
        f.write('}};')

        f.close()

    # TODO: do not use relative paths...
    with open(out_h_path, 'w+') as f:
        f.write('#ifndef _AUTOGEN_LANG_H\n')
        f.write('#define _AUTOGEN_LANG_H\n')

        f.write('#define BUCKETS_NUM {}\n'.format(num_buckets))
        f.write('#define LOCALE_NUM_LANGUAGES {}\n'.format(num_langs))

        f.write('typedef enum locale {\n')
        for l in extracted_langs:
            f.write('\tLOCALE_{},\n'.format(l))

        f.write('} jlocale_t;\n')
        f.write('#endif //_AUTOGEN_LANG_H\n')
        f.close()


#!/usr/bin/python3
"""
Fabric script based on the file 2-do_deploy_web_static.py that creates and
distributes an archive to the web servers
"""

import os.path
from fabric.api import *
from fabric.operations import run, put

env.user = 'ubuntu'
env.hosts = ['xx.xx.xx.xx', 'xx.xx.xx.xx']

def do_pack():
    """Create a .tgz archive from the contents of the web_static folder"""
    try:
        if not os.path.exists("versions"):
            local("mkdir versions")
        created_at = datetime.now().strftime("%Y%m%d%H%M%S")
        file_path = "versions/web_static_{}.tgz".format(created_at)
        local("tar -cvzf {} web_static".format(file_path))
        return file_path
    except:
        return None

def do_deploy(archive_path):
    """Distributes an archive to your web servers"""
    if not os.path.exists(archive_path):
        return False
    try:
        file_name = archive_path.split("/")[-1]
        no_ext = file_name.split(".")[0]
        path_release = "/data/web_static/releases/{}/".format(no_ext)
        put(archive_path, "/tmp/")
        run("mkdir -p {}".format(path_release))
        run("tar -xzf /tmp/{} -C {}".
            format(file_name, path_release))
        run("rm /tmp/{}".format(file_name))
        run("mv {}web_static/* {}".format(path_release, path_release))
        run("rm -rf {}web_static".format(path_release))
        run("rm -rf /data/web_static/current")
        run("ln -s {} /data/web_static/current".
            format(path_release))
        return True
    except:
        return False

def deploy():
    """Call the do_pack() function and store the path of the created archive.
    Call the do_deploy(archive_path) function, using the new path of the new archive"""
    archive_path = do_pack()
    if not archive_path:
        return False
    return do_deploy(archive_path)


from django.core.exceptions import ValidationError
from django.core.validators import BaseValidator
import jsonschema

class JSONSchemaValidator(BaseValidator):
    def compare(self, value, schema):
        try:
            jsonschema.validate(value, schema)
        except jsonschema.exceptions.ValidationError:
            ValidationError(
                f"{value} is not a valid JSON schema",
                code="invalid",
                params={"value": value},
            )


import sys
import re
import pdb
import random

DOCSTART = "0001:(U !ARTICLE)"
LOGFILE = open("numberededitabletrees2conll.log","w+")

class CorefTracker:
    def __init__(self):
        self.wordlist = [] #e.g., [('word1','0101'),('word2','0102'),...]
        self.chains = {} #e.g., {'0102':['0102','0112',...],...}
        self.article_id = 0
        self.filehandle = sys.stdout

    def printFooter(self):
        print("#end document",file = self.filehandle)

    def printHeader(self):
        print("#begin document ({});".format(self.article_id), file = self.filehandle)

    def articleIncrement(self):
        self.article_id += 1

    def clear(self):
        self.wordlist = []
        self.chains = {}
        #self.filehandle.close()

    def getWord(self, line):
        match = re.search(r"\s([^\) ]+)\)+$",line) #space, anything not a space or close paren, close paren
        assert match is not None
        return match.group(1)

    def getId(self, line):
        return str(self.article_id).zfill(3) + line.split(":")[0] #assumes zero-padded to 4 places, but annotation is sometimes not - fix annotation

    def addWord(self, line):
        id = line.split(":")[0] #assumes 2 digit sentid, 2 digit word id.
        id = str(self.article_id).zfill(3) + id #prepend article id
        word = self.getWord(line)

        self.wordlist += [(id,word)]

    def findChainId(self, annot):
        #find if it's in an existing chain
        for chainid in self.chains:
            if annot in self.chains[chainid]:
                return chainid
        return None

    def processCoref(self, line):
        #check for anaphoricity
        result = re.search(r".*-[nm]([0-9]+)", line)
        if result is not None:
            annot = str(self.article_id).zfill(3) + result.group(1).zfill(4)
            chainid = self.findChainId(annot)
            if chainid is not None:
                self.chains[chainid] += [self.getId(line)]#append current id to that chain
            else: #create new chain
                self.chains[annot] = [annot,self.getId(line)]

    def printCoNLL(self):
        if self.wordlist is [] or self.article_id == 0:
            return
        else:
            self.printHeader()
            #pdb.set_trace()
            for item in self.wordlist:
                word = item[1]
                wordnum = item[0][-2:]
                corefid = self.findChainId(item[0])
                if corefid == None:
                    corefid = "-"
                else:
                    corefid = "("+corefid+")"
                print("{}\t{}\t\t{}\t0\t{}".format(item[0], word, wordnum, corefid), file=self.filehandle)
            self.printFooter()

ct = CorefTracker()

with open(sys.argv[1],'r') as iff:
    for line in iff:
        if line.startswith(DOCSTART):
            print("found article boundary",file=LOGFILE)
            print("coref tracker contains {} chains".format(len(ct.chains)),file=LOGFILE)
            ct.printCoNLL() #print chains for previous article
            ct.clear()
            ct.articleIncrement()
            continue
        ct.addWord(line)
        ct.processCoref(line)
    ct.printCoNLL()
    ct.filehandle.close()
    LOGFILE.close()


'''
DashInsert 함수는 숫자로 구성된 문자열을 입력받은 뒤
문자열 안에서 홀수가 연속되면 두 수 사이에 - 를 추가하고,
짝수가 연속되면 * 를 추가하는 기능을 갖고 있다.
DashInsert 함수를 완성하시오.

입력 예시: 4546793
출력 예시: 454*67-9-3
'''

def dashInsert(s):
    n = 0
    result = []
    num_list = list(map(lambda x: int(x), list(s)))
    while n+1 <= len(s)-1:
        result.append(str(num_list[n]))
        is_odd = num_list[n] % 2 == 1
        is_next_odd = num_list[n + 1] % 2 == 1
        if not is_odd and not is_next_odd:  # 짝수연속
            result.append('*')
        elif is_odd and is_next_odd:  # 홀수연속
            result.append('-')
        n += 1
    result.append(str(num_list[n]))
    print("".join(result))

dashInsert('4546793')

'''

data = "4546793"
numbers = list(map(int, data))   # 숫자 문자열을 숫자 리스트로 변경
result = []

for i, num in enumerate(numbers):
    result.append(str(num))
    if i < len(numbers)-1:                   # 다음 수가 있다면
        is_odd = num % 2 == 1                # 현재 수가 홀수
        is_next_odd = numbers[i+1] % 2 == 1  # 다음 수가 홀수
        if is_odd and is_next_odd:           # 연속 홀수
            result.append("-")
        elif not is_odd and not is_next_odd: # 연속 짝수
            result.append("*")

print("".join(result))

'''


# flake8: noqa
from textx.metamodel import metamodel_from_file, metamodel_from_str
from textx.model import get_children_of_type, get_parent_of_type, \
    get_model, get_metamodel, get_children, get_location, textx_isinstance, \
    textxerror_wrap
from textx.exceptions import TextXError, TextXSyntaxError, \
    TextXSemanticError, TextXRegistrationError
from textx.registration import (LanguageDesc, GeneratorDesc,
                                register_language, register_generator,
                                language_descriptions, language_description,
                                generator_descriptions, generator_description,
                                clear_language_registrations,
                                clear_generator_registrations,
                                languages_for_file,
                                language_for_file,
                                metamodel_for_language,
                                metamodel_for_file,
                                metamodels_for_file,
                                generator_for_language_target,
                                generator, language)

__version__ = "3.2.0.dev"


import re
import requests
from bs4 import BeautifulSoup
from redis import StrictRedis, ConnectionPool

def get_urls(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.content
        else:
            print('request error code:', response.status_code)
    except Exception as e:
        print(e)
        return None

def parse(input, cutLetter):
    re_tag = re.compile('</?\w+[^>]*>')
    content = re_tag.sub('', str(input))
    if cutLetter:
        re_useless = re.compile('[a-zA-Z,]')
        return re_useless.sub('', content)
    return content

def getRedisConnectionPool():
    url = 'redis://@localhost:6379/0'
    pool = ConnectionPool.from_url(url)
    return pool

def main():
    headers = {
        'user-agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
    }

    url_co_prefix = 'https://www.youtube.com/watch?v='

    url_ids = {
        'hitomi': [
            'MwEU_09TaLs','nXjKqkAJ-R8','4lZx2f-TbY0','rftFz2tUbKE','VFgpwfEIrnI',
            'u-GRWRs_d6I','yQFzeBzO_ZE','zfaGTwmxG3A','kMYEntl-_js','mVH_dzY3wRI',
            'IH7cIEByWWU','t_LcpM-QG-4','0hr-DyTWLU4','9eIfStw-e_c','1r6fPvCt1UE',
            'XCZEnJ5L81Q','KScSFIrl8HQ','txH9eX8hD_Y','bxWD4TpxKWI','fwrQE9pGizM',
            'j25j272Z9hQ','jFPz8wsQurw','7IFWkA0FM9o',
        ],
        'nako': [
            'vIcFnNdHc2w', '5YHLH9CVff0', 'o6VpbfOT9R0', 'Rry_viMzp_M',
            'rfvYiRUeK0I', 'GSVurT6v6gs', 'Lq7rgg_Rq7E', 'ucg-KHmPrXY',
            'UQD62p2TDqQ', 'PR6ZdZZiQNE', 'A-3YKK1NNJ4', 'VtxvZq6dB3Q',
            'atI1ZzBArrs', '6R5ORUUiLRQ', '-l7etfJDAso', 'MfdCxncxbQw',
            'vqHLAVx40t0', 'c-IP3n-VUm0', '-xPD5CovM70', 'DAEph5aJGlQ'
        ],
        'sakura': [
            'm2K26HTAi_E', 'eWdxzvTy88k', 'YeJ9s9Xtzug', 'qp7dHeGPi08',
            'nKcXsTOIesE', 'ywEkHQyxevQ', 'sDBPKy5jHRY', 'tnDHnbk1Pds',
            'T2Bvxr2OTUU', 'Y3RkiW0eWwQ', 'idgk0b0lioo', 'XIbrYlvCvE8',
            'DOF0M6KKw8E', 'KDfebdR8XyM', 'U_vNFe1XrlM', 'ks6fbGC2SP8',
            'Upi90X8DlbU', 'GuU_ob0diSE', '3fzXLGMFWRo', 'hdIf6j4O5TQ',
            's5oc82ZPyEs', 'nSMt0IPwAhQ', '3EkshpX5_AQ', 'f-ehRuVT19k',
            'V0WWskPWmLQ', 'W0hntlJHAXY', 'evhL6ZE4Iq0', 'tXojxAwO2R0',
            'gGpNfVCIkxU', 'zAvvJ8OxVlk', '5HjjumzgNtY', 'MXqG0e6wuY4',
            'bDAoIX1nyjY', 'xPhSvAJvbVA', 'M17XCxxAWGs', 'hcSBFdMbWVk',
            '25bEVyeTg9c', 'Hpp1oMorYNI', 'waWpOPLZPMU'
        ],
        'nayeon': [
            '6fbxCnIyS4c', 'O4XEJhLh124', 'KQiqzFmKti0', '2I8hTLs9B8c',
            'TqhQWwV9hSc', 'fhyLpq-7OL0', 'Rwyr4Q6tD4Q', 'FL6TpPBqTZk',
            'Vq2q6cLQIvc', '3iu0Y8UbRw0', 'rc35kimy7VU', 'IpvZrVPH_NE',
            'xquaIy535_c', 'ZdWfXyX-dxQ', '-RcUOYxo3tk', 'H1ky7IKp914',
            '838Bj2TTxOA'
        ]
    }

    pool = getRedisConnectionPool()
    redis = StrictRedis(connection_pool=pool)

    for key in url_ids.keys():
        ids = url_ids[key]
        print(key + ":")
        for id in ids:
            html = get_urls(url_co_prefix + id)
            soup = BeautifulSoup(html, 'html.parser')
            title = parse(soup.head.title, False)
            content = soup.find('div', {'class': 'watch-view-count'})
            viewCount = re.findall(r'\d+', parse(content, True))[0]
            delta = int(viewCount)
            if redis.exists(key, title):
                preViewCount = int(redis.hget(key, title).decode('utf-8'))
                delta = delta - preViewCount
            redis.hset(key, title, str(viewCount))
            print(delta, viewCount, title)

if __name__ == "__main__":
    main()


from collections import namedtuple
import json
import warnings

EntityField = namedtuple("EntityField", ["name", "isOptional"])

class WhatIfValidator():
    """
    Provides a simple way to validate that your entities will successfully
    upload.  Provides functions to validate the type, check if required
    attributes are missing, and check if superfluous attributes are included.

    :param dict type_defs:
        The list of type definitions to be validated against.  Should be
        in the form of an AtlasTypeDef composite wrapper.
    :param list(dict) existing_entities:
        The existing entities that should be validated against.
    """
    ASSET_ATTRIBUTES = ["name", "description", "owner"]
    REFERENCABLE_ATTRIBUTES = ["qualifiedName"]

    ATLAS_MODEL = {
        "ASSET": ["name", "description", "owner"],
        "REFERENCABLE": ["qualifiedName"],
        "PROCESS": (["inputs", "outputs"] +
                    ASSET_ATTRIBUTES + REFERENCABLE_ATTRIBUTES),
        "DATASET": ASSET_ATTRIBUTES + REFERENCABLE_ATTRIBUTES,
        "INFRASTRUCTURE": ASSET_ATTRIBUTES + REFERENCABLE_ATTRIBUTES
    }

    def __init__(self, type_defs={}, existing_entities=[]):
        if len(type_defs) == 0 and len(existing_entities) == 0:
            warnings.warn(
                "WARNING: Provided type_defs and existing_entities are empty.  All validations will pass.")

        self.classification_defs = type_defs.get("classificationDefs", [])
        self.entity_defs = type_defs.get("entityDefs", [])
        # Create a dict of all entities by name, then find the name of all
        # attributes and whether they are optional
        entity_fields = {}
        for e in self.entity_defs:
            entity_fields[e["name"]] = []
            for attr in e.get("attributeDefs", []):
                ef = EntityField(attr.get("name"), attr.get("isOptional"))
                entity_fields[e["name"]].append(ef)

        # Adding Qualified Name to the set of valid fields as it doesn't
        # show up in the entity type def
        self.entity_valid_fields = {k: set(
            [field.name for field in v] + ["qualifiedName"])
            for k, v in entity_fields.items()}
        # Adding Qualified Name to the set of required entity
        # fields as it doesn't show up in the entity type def
        self.entity_required_fields = {
            k: set([field.name for field in v if not field.isOptional] + [
                "qualifiedName"]) for k, v in entity_fields.items()
        }

        self.existing_entities = set(
            [e.get("attributes", {}).get("qualifiedName") for
             e in existing_entities])

        self.enum_defs = type_defs.get("enumDefs", [])
        self.relationship_defs = type_defs.get("relationshipDefs", [])
        self.struct_defs = type_defs.get("structDefs", [])

    def entity_type_exists(self, entity):
        """
        Validate that the entity's type is an actual entity definition.

        :param dict entity:
        :return: Whether the entity matches the list of known entity types.
        :rtype: bool
        """
        current_type = entity["typeName"]
        if current_type in self.entity_valid_fields:
            return True
        else:
            return False

    def entity_missing_attributes(self, entity):
        """
        Check if the entity is missing required attributes.

        :param dict entity:
        :param dict type_def:
        :return: Whether the entity matches the list of known entity types.
        :rtype: bool
        """

        current_attributes = set(entity.get("attributes", {}).keys())
        required_attributes = set(
            self.entity_required_fields[entity["typeName"]])
        missing_attributes = required_attributes.difference(current_attributes)
        if len(missing_attributes) > 0:
            return missing_attributes
        else:
            return False

    def entity_has_invalid_attributes(self, entity):
        """
        Check if the entity is using attributes that are not defined
        on the type.

        :param dict entity:
        :return: Whether the entity matches the list of known entity types.
        :rtype: bool
        """
        current_attributes = set(entity.get("attributes", {}).keys())
        valid_attributes = set(self.entity_valid_fields[entity["typeName"]])
        # Append inherited attributes:
        _entity_type = entity["typeName"]
        # Assuming only one entity matches and only one super type
        super_type = [e["superTypes"]
                      for e in self.entity_defs
                      if e["name"] == _entity_type][0][0].upper()
        if super_type in self.ATLAS_MODEL:
            valid_attributes = valid_attributes.union(
                self.ATLAS_MODEL[super_type])
        invalid_attributes = current_attributes.difference(valid_attributes)

        if len(invalid_attributes) > 0:
            return invalid_attributes
        else:
            return False

    def entity_would_overwrite(self, entity):
        """
        Based on the qualified name attributes, does the provided
        entity exist in the entities provided to the What If Validator?

        :param dict entity:
        :return: Whether the entity matches an existing entity.
        :rtype: bool
        """
        current_qualifiedName = entity.get(
            "attributes", {}).get("qualifiedName")
        if current_qualifiedName is None:
            raise KeyError("Entity has no qualifiedName:\n{}".format(
                json.dumps(entity, indent=1)))

        if current_qualifiedName in self.existing_entities:
            return True
        else:
            return False

    def validate_entities(self, entities):
        """
        Provide a report of invalid entities.  Includes TypeDoesNotExist,
        UsingInvalidAttributes, and MissingRequiredAttributes.

        :param list(dict) entities: A list of entities to validate.
        :return: A dictionary containing counts values for the above values.
        :rtype: dict
        """
        report = {"TypeDoesNotExist": [], "UsingInvalidAttributes": {},
                  "MissingRequiredAttributes": {}}

        for entity in entities:
            if not self.entity_type_exists(entity):
                report["TypeDoesNotExist"].append(entity["guid"])
                # If it's an invalid type, we have to skip over the rest
                # of this
                continue

            using_invalid = self.entity_has_invalid_attributes(entity)
            is_missing = self.entity_missing_attributes(entity)
            cur_guid = entity["guid"]
            if using_invalid:
                report["UsingInvalidAttributes"][cur_guid] = using_invalid

            if is_missing:
                report["MissingRequiredAttributes"][cur_guid] = is_missing

        output = {
            "counts": {k: len(v) for k, v in report.items()},
            "values": report
        }
        output.update({"total": sum(output["counts"].values())})

        return output


import pandas as pd

def get_data():
    numbers = [13, 14, 15, 16, 17, 18]
    persons = ["Manuel", "pedro", "fernando", "maria", "antônia", "joana"]

    dict_to_df = {"names": persons, "numbers": numbers}

    df = pd.DataFrame(dict_to_df)

    return df


from mbot.modeling.datasets.facebook_dataset import FacebookDataset
from mbot.modeling.datasets.cornell_dataset import CornellDataset
from mbot.modeling.datasets.combined_dataset import CombinedDataset
from mbot.modeling.models.preprocessor import Preprocessor
from mbot.modeling.models.seq2seq import Seq2Seq

__author__ = 'Elisha Yadgaran'

DATASETS = [FacebookDataset, CornellDataset]

def create_dataset(datasets, load_raw_data=False):
    def get_dataset_class(name):
        for i in DATASETS:
            if i.__name__ == name:
                return i

    dataset_classes = [get_dataset_class(i) for i in datasets]
    datasets = [dataset(load_raw_data) for dataset in dataset_classes]
    dataset = CombinedDataset(datasets)

    return dataset

def preprocess_dataset(dataset):
    preprocessor = Preprocessor()
    preprocessor.fit(dataset)
    preprocessor.save()

    return preprocessor

def create_model(preprocessor):
    model = Seq2Seq(preprocessor)
    model.compile_model()

    return model

if __name__ == '__main__':
    dataset = create_dataset(['FacebookDataset', 'CornellDataset'])
    preprocessor = preprocess_dataset(dataset)
    model = create_model(preprocessor)
    model.train(dataset.x, dataset.y)
    model.save()


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import os
import json

class Inventory:
    def _init_(self):
        #AT LAUNCH GROUPS AND LOADING FUNCTION
        self.items = {}
        self.load()

    def remove(self, ID):
        #REMOVING ITEMS FOR LISTS AND OUTPUT DOCUMENT
        del self.items[str(ID)]
        self.save()

    def add(self, ID, name, qty):
        #ADDING ITEMS FOR LISTS AND OUTPUT DOCUMENT
        self.items[ID] = {"name": name, "qty": qty}
        self.save()

    def update(self, ID, update):
        #UPDATING ITEMS FOR LISTS AND OUTPUT DOCUMENT
        self.items[ID]["qty"] += update
        self.save()

    def search(self, ID):
        #SEARCHING ITEMS FOR LISTS
        item = self.items.get(ID, None)
        if item:
            return ID, item['name'], item['qty']
        else:
            return None

    def _str_(self):
        #FORMATTING
        out = ""
        for id, d in self.items.items():
            out += f"ID Number : {id} \nItem Name : {d['name']}\nQuantity : {d['qty']}\n"
            out += "----------\n"
        return out

    def save(self):
        #WHERE TO SAVE TO
        with open('data.txt','w') as outfile:
           json.dump(self.items, outfile)

    def load(self):
        #WHERE TO PUT DATA FROM WHEN RELAUNCHING PROGRAM
        try:
            with open('data.txt','r') as json_file:
               self.items = json.load(json_file)
        except:
            print("Can't load old inventory, starting fresh")
            self.items = {}

def menuDisplay():
    #MENU FOR PROGRAM
    """Display the menu"""
    print('=============================')
    print('= Inventory Management Menu =')
    print('=============================')
    print('(1) Add New Item to Inventory')
    print('(2) Remove Item from Inventory')
    print('(3) Update Inventory')
    print('(4) Search Item in Inventory')
    print('(5) Print Inventory Report')
    print('(99) Quit')

def add_one_item(inventory):
    #ADDING PROMPT AND ERROR CHECKING
    print('Adding Inventory')
    print('================')
    while True:
        try:
            new_ID = int(input("Enter an ID number for the item: "))
            if inventory.search(new_ID):
                print("ID number is taken, please enter a different ID number")
                continue
            new_name = input('Enter the name of the item: ').lower()
            assert new_name.isalpha(), "Only letters are allowed!"
            new_qty = int(input("Enter the quantity of the item: "))
            inventory.add(new_ID, new_name, new_qty)
            break
        except Exception as e:
            print("Invalid choice! try again! " + str(e))
            print()

def remove_one_item(inventory):
    #REMOVING PROMPT AND ERROR CHECKING
    print('Removing Inventory')
    print('==================')
    removing = int(input("Enter the item's ID number to remove from inventory: "))
    inventory.remove(removing)

def ask_exit_or_continue():
    #OPTION TO CONTINUE OR QUITE PROGRAM
    return int(input('Enter 98 to continue or 99 to exit: '))

def update_inventory(inventory):
    #UPDATING PROMPT AND ERROR CHECKING
    print('Updating Inventory')
    print('==================')
    ID = int(input("Enter the item's ID number to update: "))
    update = int(input("Enter the updated quantity. Enter 5 for additional or -5 for less: "))
    inventory.update(ID, update)

def search_inventory(inventory):
    #SEARCHING PROMPT AND ERROR CHECKING
    print('Searching Inventory')
    print('===================')
    search = int(input("Enter the ID number of the item: "))
    result = inventory.search(search)
    if result is None:
        print("Item not in inventory")
    else:
        ID, name, qty = result
        print('ID Number: ', ID)
        print('Item:     ', name)
        print('Quantity: ', qty)
        print('----------')

def print_inventory(inventory):
    #PRINT CURRENT LIST OF ITEMS IN INVENTORY
    print('Current Inventory')
    print('=================')
    print(inventory)

def main():
    #PROGRAM RUNNING COMMAND AND ERROR CHECKING
    inventory = Inventory()
    while True:
        try:
            menuDisplay()
            CHOICE = int(input("Enter choice: "))
            if CHOICE in [1, 2, 3, 4, 5]:
                if CHOICE == 1:
                    add_one_item(inventory)
                elif CHOICE == 2:
                    remove_one_item(inventory)
                elif CHOICE == 3:
                    update_inventory(inventory)
                elif CHOICE == 4:
                    search_inventory(inventory)
                elif CHOICE == 5:
                    print_inventory(inventory)
                exit_choice = ask_exit_or_continue()
                if exit_choice == 99:
                    exit()
            elif CHOICE == 99:
                exit()
        except Exception as e:
            print("Invalid choice! try again!"+str(e))
            print()

        # If the user pick an invalid choice,
        # the program will come to here and
        # then loop back.

main()


import torch
from torch.nn import Module, ModuleList, Conv1d, Sequential, ReLU, Dropout
from torch_geometric.nn import Linear, EdgeConv, GATv2Conv, SAGEConv, BatchNorm

class DilatedResidualLayer(Module):
    def __init__(self, dilation, in_channels, out_channels):
        super(DilatedResidualLayer, self).__init__()
        self.conv_dilated = Conv1d(in_channels, out_channels, kernel_size=3, padding=dilation, dilation=dilation)
        self.conv_1x1 = Conv1d(out_channels, out_channels, kernel_size=1)
        self.relu = ReLU()
        self.dropout = Dropout()

    def forward(self, x):
        out = self.relu(self.conv_dilated(x))
        out = self.conv_1x1(out)
        out = self.dropout(out)
        return x + out

# This is for the iterative refinement (we refer to MSTCN++: https://github.com/sj-li/MS-TCN2)
class Refinement(Module):
    def __init__(self, final_dim, num_layers=10, interm_dim=64):
        super(Refinement, self).__init__()
        self.conv_1x1 = Conv1d(final_dim, interm_dim, kernel_size=1)
        self.layers = ModuleList([DilatedResidualLayer(2**i, interm_dim, interm_dim) for i in range(num_layers)])
        self.conv_out = Conv1d(interm_dim, final_dim, kernel_size=1)

    def forward(self, x):
        f = self.conv_1x1(x)
        for layer in self.layers:
            f = layer(f)
        out = self.conv_out(f)
        return out

class SPELL(Module):
    def __init__(self, cfg):
        super(SPELL, self).__init__()
        self.use_spf = cfg['use_spf'] # whether to use the spatial features
        self.use_ref = cfg['use_ref']
        self.num_modality = cfg['num_modality']
        channels = [cfg['channel1'], cfg['channel2']]
        final_dim = cfg['final_dim']
        num_att_heads = cfg['num_att_heads']
        dropout = cfg['dropout']

        if self.use_spf:
            self.layer_spf = Linear(-1, cfg['proj_dim']) # projection layer for spatial features

        self.layer011 = Linear(-1, channels[0])
        if self.num_modality == 2:
            self.layer012 = Linear(-1, channels[0])

        self.batch01 = BatchNorm(channels[0])
        self.relu = ReLU()
        self.dropout = Dropout(dropout)

        self.layer11 = EdgeConv(Sequential(Linear(2*channels[0], channels[0]), ReLU(), Linear(channels[0], channels[0])))
        self.batch11 = BatchNorm(channels[0])
        self.layer12 = EdgeConv(Sequential(Linear(2*channels[0], channels[0]), ReLU(), Linear(channels[0], channels[0])))
        self.batch12 = BatchNorm(channels[0])
        self.layer13 = EdgeConv(Sequential(Linear(2*channels[0], channels[0]), ReLU(), Linear(channels[0], channels[0])))
        self.batch13 = BatchNorm(channels[0])

        if num_att_heads > 0:
            self.layer21 = GATv2Conv(channels[0], channels[1], heads=num_att_heads)
        else:
            self.layer21 = SAGEConv(channels[0], channels[1])
            num_att_heads = 1
        self.batch21 = BatchNorm(channels[1]*num_att_heads)

        self.layer31 = SAGEConv(channels[1]*num_att_heads, final_dim)
        self.layer32 = SAGEConv(channels[1]*num_att_heads, final_dim)
        self.layer33 = SAGEConv(channels[1]*num_att_heads, final_dim)

        if self.use_ref:
            self.layer_ref1 = Refinement(final_dim)
            self.layer_ref2 = Refinement(final_dim)
            self.layer_ref3 = Refinement(final_dim)

    def forward(self, x, edge_index, edge_attr, c=None):
        feature_dim = x.shape[1]

        if self.use_spf:
            x_visual = self.layer011(torch.cat((x[:, :feature_dim//self.num_modality], self.layer_spf(c)), dim=1))
        else:
            x_visual = self.layer011(x[:, :feature_dim//self.num_modality])

        if self.num_modality == 1:
            x = x_visual
        elif self.num_modality == 2:
            x_audio = self.layer012(x[:, feature_dim//self.num_modality:])
            x = x_visual + x_audio

        x = self.batch01(x)
        x = self.relu(x)

        edge_index_f = edge_index[:, edge_attr<=0]
        edge_index_b = edge_index[:, edge_attr>=0]

        # Forward-graph stream
        x1 = self.layer11(x, edge_index_f)
        x1 = self.batch11(x1)
        x1 = self.relu(x1)
        x1 = self.dropout(x1)
        x1 = self.layer21(x1, edge_index_f)
        x1 = self.batch21(x1)
        x1 = self.relu(x1)
        x1 = self.dropout(x1)

        # Backward-graph stream
        x2 = self.layer12(x, edge_index_b)
        x2 = self.batch12(x2)
        x2 = self.relu(x2)
        x2 = self.dropout(x2)
        x2 = self.layer21(x2, edge_index_b)
        x2 = self.batch21(x2)
        x2 = self.relu(x2)
        x2 = self.dropout(x2)

        # Undirected-graph stream
        x3 = self.layer13(x, edge_index)
        x3 = self.batch13(x3)
        x3 = self.relu(x3)
        x3 = self.dropout(x3)
        x3 = self.layer21(x3, edge_index)
        x3 = self.batch21(x3)
        x3 = self.relu(x3)
        x3 = self.dropout(x3)

        x1 = self.layer31(x1, edge_index_f)
        x2 = self.layer32(x2, edge_index_b)
        x3 = self.layer33(x3, edge_index)

        out = x1+x2+x3

        if self.use_ref:
            xr0 = torch.permute(out, (1, 0)).unsqueeze(0)
            xr1 = self.layer_ref1(torch.softmax(xr0, dim=1))
            xr2 = self.layer_ref2(torch.softmax(xr1, dim=1))
            xr3 = self.layer_ref3(torch.softmax(xr2, dim=1))
            out = torch.stack((xr0, xr1, xr2, xr2), dim=0).squeeze(1).transpose(2, 1).contiguous()

        return out


from celery import shared_task
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)
from .models import Counter

@shared_task(bind=True)
def add_first(self):
    counter = Counter.objects.first()
    counter.number = counter.number + 20
    counter.save()
    logger.info(str(counter.number))
    # logger.info("first task celery bright")
    return 'done'


from graphframes import *
from pyspark.sql import SQLContext
from pyspark import SparkContext

# Create DataFrames from HIVE query
v=SQLContext.sql("select  id, name, total_seconds from my_schema.nodes")
e=SQLContext.sql("select src, dst, relationship from my_schema.edges")

# Create DataFrames manually for testing purposes
v = SQLContext.createDataFrame([
    ("A", "ARON"  ,350 ),
    ("B", "BILL"  ,360 ),
    ("C", "CLAIR" ,195 ),
    ("D", "DANIEL",90),
    ("E", "ERIC"  ,90),
    ("F", "FRANK" ,215 ),
    ("G", "GRAHAM",30 ),
    ("H", "HENRY" ,25 ),
    ("I", "INNA"  ,25 ),
    ("J", "JEN"   ,20 )
], ["id", "name", "total_seconds"])

e=SQLContext.createDataFrame([
    ("A", "B", 60),
    ("B", "A", 50),
    ("A", "C", 50),
    ("C", "A", 100),
    ("A", "D", 90),
    ("C", "I", 25),
    ("C", "J", 20),
    ("B", "F", 50),
    ("F", "B", 110),
    ("F", "G", 30),
    ("F", "H", 25),
    ("B", "E", 90)
],["src","dst","relationship"])

# Now lets construct the graph
g = GraphFrame(v,e)


def twoSum(nums: list[int], target: int) -> list[int]:
        prevMap = {} # val : index

        for i, n in enumerate(nums):
            diff = target - n
            if diff in prevMap:
                return [prevMap[diff], i]
            prevMap[n] = i
        return

nums = [2,7,11,15]
target = 9
print(twoSum(nums,target))

def twoSum2(numbers, target):
        """
        :type numbers: List[int]
        :type target: int
        :rtype: List[int]
        """
        d = {}
        for i in range(len(numbers)):
            t = target - numbers[i]
            if t in d:
                return[d[t]+1,i+1]
            else:
                d[numbers[i]] = i

print(twoSum2(nums,target))


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# leiyh

'''
4、使用列表推导式+join方法把下列字典转成键值对
'''
d = {"username": 'xuepl', "sex": '男', "age": 18}
d = [keys + "=" + str(values) for keys, values in d.items()]
print("&".join(d))

'''
5、使用列表推导式过滤掉长度小于3的字符串，并将剩下的字符串转换成大写字母
'''
lst=["1","2","3","4","8","weeff","bdskjk","nsdjj"]
lst1 = [i.upper() for i in lst if len(i)>3]
print(lst1)