# -*- coding: utf-8 -*-
"""
Created on Mon Jun  5 11:19:18 2023

@author: Dan
"""

import requests
import pandas as pd
import numpy as np
import cx_Oracle
import time

from rapidfuzz.process import extractOne
from rapidfuzz.fuzz import QRatio
from generic.generic_data_functions_common import (cal_lookup, sanitize_cal_input,
                                   parse_prod_request, parse_loc_request,
                                   prod_hier, loc_hier, product_data)
from generic.generic_data_config import (data_url, username, password, dbname, n_responses, synonyms)

# =============================================================================
#
# =============================================================================

# Helper function for calling presto-data-services apis

def call_api(api_path, locs, prods, cal, active, start_week = None):

    responses = []
    start = time.time()
    for l in locs:
        for z in locs[l]:
            for p in prods:
                if p > 1:
                    for w in prods[p]:
                        temp = prod_hier(product_id = w, product_level = p, detail_level = 1, active = active)
                        if p >= 2:
                            response = requests.post(data_url + api_path,
                                                     json = {"locationLevelId": int(l),
                                                             "locationId": int(z),
                                                             "startDate": start_week.strftime('%m/%d/%Y') if start_week else cal['end-date'].min().strftime('%m/%d/%Y'),
                                                             "endDate": cal['end-date'].max().strftime('%m/%d/%Y'),
                                                             "calType": "W",
                                                             "fiscalOrAdCalendar": "B",
                                                             "productId":int(w),
                                                             "productLevelId":int(p)})
                        elif p == 1.5:
                            response = requests.post(data_url + api_path,
                                                     json = {"locationLevelId": int(l),
                                                             "locationId": int(z),
                                                             "startDate": start_week.strftime('%m/%d/%Y') if start_week else cal['end-date'].min().strftime('%m/%d/%Y'),
                                                             "endDate": cal['end-date'].max().strftime('%m/%d/%Y'),
                                                             "calType": "W",
                                                             "fiscalOrAdCalendar": "B",
                                                             "itemCodes":[int(x) for x in temp['item-code']]})
                        response = pd.DataFrame(response.json())
                        if response.empty:
                            continue
                        response = response.rename(columns = {'itemCode':'item-code'})
                        response = temp.loc[:, ['item-name', 'item-code']].merge(response, how = 'inner', on = 'item-code')
                        response['product-level'] = p
                        response['product-id'] = w
                        responses.append(response)
                elif p == 1:
                    temp = prod_hier(product_id = prods[p], product_level = p, detail_level = 1, active = active)
                    response = requests.post(data_url + api_path,
                                             json = {"locationLevelId": int(l),
                                                     "locationId": int(z),
                                                     "startDate": start_week.strftime('%m/%d/%Y') if start_week else cal['end-date'].min().strftime('%m/%d/%Y'),
                                                     "endDate": cal['end-date'].max().strftime('%m/%d/%Y'),
                                                     "calType": "W",
                                                     "fiscalOrAdCalendar": "B",
                                                     "itemCodes":[int(x) for x in temp['item-code']]})
                    response = pd.DataFrame(response.json())
                    if response.empty:
                        continue
                    response = response.rename(columns = {'itemCode':'item-code'})
                    response = temp.loc[:, ['item-name', 'item-code']].merge(response, how = 'inner', on = 'item-code')
                    response['product-level'] = p
                    response['product-id'] = response['item-code']
                    responses.append(response)
    response = pd.concat(responses, ignore_index = True)
    end = time.time()
    print('Time taken: {} seconds'.format(round(end - start, 2)))
    return response

# =============================================================================
#
# =============================================================================

# Default calendar behavior if LLM fails to specify.

def cal_logic(cal_year, quarter, period, week, day, start_date, end_date, calendar_id):
    if start_date is not None or end_date is not None or calendar_id is not None:
        return cal_year, quarter, period, week, day, start_date, end_date, calendar_id
    if cal_year is None:
        if quarter is None and period is None and week is None and day is None:
            quarter = 'current'
        cal_year, quarter, period, week, day = sanitize_cal_input(cal_year, quarter, period, week, day)
        if type(quarter) == int or type(quarter) == list:
            cal_year = 'current'
        if type(period) == int or type(period) == list:
            cal_year = 'current'
        if type(week) == int or type(week) == list:
            cal_year = 'current'
        if type(day) == int or type(day) == list:
            cal_year = 'current'
    return cal_year, quarter, period, week, day, start_date, end_date, calendar_id

# =============================================================================
#
# =============================================================================

# Sometimes LLM gives empty list, e.g. [] or list with something strange like ['all']. This catches and fixes its mistakes.
def catch_empty_list(l):
    if type(l) == list:
        l_2 = [x for x in l if x.lower() not in ['all', 'each', 'every']]
        if len(l_2) > 0:
            return l_2
        else:
            return None
    else:
        return l

# =============================================================================
#
# =============================================================================

# Not currently used.
def sanitize_list(x):
    if type(x) == list and len(x) > 0:
        return x[0] if x[0].lower() not in ['all', 'every', 'each'] else None
    return x

# =============================================================================
#
# =============================================================================

# Fetches cost data. First it, parses timeframe, product, and location.
# Then it calls the cost api from presto-data-services. Finally, it formats
# the result.

def cost_api(product_name = None, product_id = None, product_level = None, item_list = None,
             active = 'Y', group_name = None,
             location_name = None, location_id = None, location_level = None,
             cal_year = None, quarter = None, period = None, week = None, day = None,
             start_date = None, end_date = None, calendar_id = None, cal_type = 'W', user_id = None, change = 'All'):

    cal_year, quarter, period, week, day, start_date, end_date, calendar_id = cal_logic(cal_year, quarter, period, week, day, start_date, end_date, calendar_id)

    cal = cal_lookup(cal_year, quarter, period, week, day,
                     start_date, end_date, calendar_id, cal_type)

    if cal.empty:
        raise ValueError('timeframe')

    last_week = cal_lookup(week = 'last 1')
    start_week = min(cal['start-date'].min(), last_week['start-date'].min())

    location_name = catch_empty_list(location_name)

    locations = parse_loc_request(location_name, location_id, location_level, active)

    if locations.empty:
        raise ValueError('location')

    loc_names = list(locations['location-name'].drop_duplicates())
    locs = {l:[int(x) for x in locations.loc[locations['location-level'] == l, 'location-id']] for l in locations['location-level'].unique()}

    product_name = catch_empty_list(product_name)

    products = parse_prod_request(product_name, product_id, product_level,
                               user_id, active, n_responses)

    if products.empty:
        raise ValueError('product')

    if (products['authorized'] == False).all():
        raise ValueError('authorization')

    products = products.loc[products['authorized']]

    prod_names = list(products['product-name'].drop_duplicates())
    prods = {p:[int(x) for x in products.loc[products['product-level'] == p, 'product-id']] for p in products['product-level'].unique()}

    try:
        response = call_api('/cost/get-cost-data', locs, prods, cal, active, start_week)
    except:
        raise ValueError('missing_data')

    response = response.rename(columns = {'calendarId':'calendar-id',
                                          'startDate':'effective-date',
                                          'locationId':'location-id',
                                          'locationLevelId':'location-level'})
    response['location-level'] = np.where(response['location-level'] == 2, 5, response['location-level'])
    response['location-level'] = np.where(response['location-level'] == 1, 6, response['location-level'])
    response['location-level'] = np.where(response['location-level'] == 0, 1, response['location-level'])
    response = response.merge(locations, how = 'left')

    items = response.loc[:, ['item-name', 'item-code']].drop_duplicates()
    items['key'] = 0
    extended_cal = pd.concat([cal, last_week] if last_week['start-date'].min() <= cal['end-date'].max() else [cal],
                             ignore_index = True).drop_duplicates()
    extended_cal['key'] = 0
    extended_cal = extended_cal.merge(items, on = 'key')
    extended_cal = extended_cal.merge(response, how = 'left', on = ['calendar-id', 'item-code', 'item-name'])
    extended_cal = extended_cal.sort_values(by = ['calendar-id', 'item-code'])
    extended_cal['cost'] = extended_cal.groupby(['item-code'])['cost'].transform(lambda x: x.fillna(method = 'ffill'))
    extended_cal['effective-date'] = extended_cal.groupby(['item-code'])['effective-date'].transform(lambda x: x.fillna(method = 'ffill'))

    extended_cal = extended_cal.loc[extended_cal['calendar-id'].isin(cal['calendar-id']),
                                    ['cal-year', 'quarter-no', 'period-no', 'week-no',
                                     'start-date', 'end-date', 'location-name',
                                     'item-name', 'item-code', 'cost', 'effective-date']]

    extended_cal['start-date'] = extended_cal['start-date'].dt.strftime('%m/%d/%Y')
    extended_cal['end-date'] = extended_cal['end-date'].dt.strftime('%m/%d/%Y')
    extended_cal = extended_cal.rename(columns = {'start-date':'week-start-date',
                                                  'end-date':'week-end-date',
                                                  'cost':'list-cost'})
    extended_cal = extended_cal.dropna()

    if change == 'Yes':
        extended_cal = extended_cal.groupby(['item-name', 'item-code']).filter(lambda x: (x['list-cost'].round(2) != x['list-cost'].round(2).min()).any())
    if change == 'No':
        extended_cal = extended_cal.groupby(['item-name', 'item-code']).filter(lambda x: (x['list-cost'].round(2) == x['list-cost'].round(2).min()).all())

    return extended_cal, prod_names, loc_names, cal['start-date'].min().strftime('%m/%d/%Y') + ' - ' + cal['end-date'].max().strftime('%m/%d/%Y')

# =============================================================================
#
# =============================================================================

# Same as above but with price instead of cost.

def price_api(product_name = None, product_id = None, product_level = None, item_list = None,
              active = 'Y', group_name = None,
             location_name = None, location_id = None, location_level = None,
             cal_year = None, quarter = None, period = None, week = None, day = None,
             start_date = None, end_date = None, calendar_id = None, cal_type = 'W', user_id = None, change = 'All'):

    cal_year, quarter, period, week, day, start_date, end_date, calendar_id = cal_logic(cal_year, quarter, period, week, day, start_date, end_date, calendar_id)

    cal = cal_lookup(cal_year, quarter, period, week, day,
                     start_date, end_date, calendar_id, cal_type)

    if cal.empty:
        raise ValueError('timeframe')

    last_week = cal_lookup(week = 'last 1')
    start_week = min(cal['start-date'].min(), last_week['start-date'].min())

    location_name = catch_empty_list(location_name)

    locations = parse_loc_request(location_name, location_id, location_level, active)

    if locations.empty:
        raise ValueError('location')

    loc_names = list(locations['location-name'].drop_duplicates())
    locs = {l:[int(x) for x in locations.loc[locations['location-level'] == l, 'location-id']] for l in locations['location-level'].unique()}

    product_name = catch_empty_list(product_name)

    products = parse_prod_request(product_name, product_id, product_level,
                               user_id, active, n_responses)

    if products.empty:
        raise ValueError('product')

    if (products['authorized'] == False).all():
        raise ValueError('authorization')

    products = products.loc[products['authorized']]

    prod_names = list(products['product-name'].drop_duplicates())
    prods = {p:[int(x) for x in products.loc[products['product-level'] == p, 'product-id']] for p in products['product-level'].unique()}

    try:
        response = call_api('/price/get-price-data', locs, prods, cal, active, start_week)
    except:
        raise ValueError('missing_data')

    response = response.rename(columns = {'calendarId':'calendar-id',
                                          'startDate':'effective-date',
                                          'locationId':'location-id',
                                          'locationLevelId':'location-level'})
    response['location-level'] = np.where(response['location-level'] == 2, 5, response['location-level'])
    response['location-level'] = np.where(response['location-level'] == 1, 6, response['location-level'])
    response['location-level'] = np.where(response['location-level'] == 0, 1, response['location-level'])
    response = response.merge(locations, how = 'left')

    items = response.loc[:, ['item-name', 'item-code']].drop_duplicates()
    items['key'] = 0
    extended_cal = pd.concat([cal, last_week] if last_week['start-date'].min() <= cal['end-date'].max() else [cal],
                             ignore_index = True).drop_duplicates()
    extended_cal['key'] = 0
    extended_cal = extended_cal.merge(items, on = 'key')
    extended_cal = extended_cal.merge(response, how = 'left', on = ['calendar-id', 'item-code', 'item-name'])
    extended_cal = extended_cal.sort_values(by = ['calendar-id', 'item-code'])

    extended_cal['price'] = extended_cal.groupby(['item-code'])['price'].transform(lambda x: x.fillna(method = 'ffill'))
    extended_cal['priceQty'] = extended_cal.groupby(['item-code'])['priceQty'].transform(lambda x: x.fillna(method = 'ffill'))
    extended_cal['effective-date'] = extended_cal.groupby(['item-code'])['effective-date'].transform(lambda x: x.fillna(method = 'ffill'))

    extended_cal = extended_cal.loc[extended_cal['calendar-id'].isin(cal['calendar-id']),
                                    ['cal-year', 'quarter-no', 'period-no', 'week-no',
                                     'start-date', 'end-date', 'location-name',
                                     'item-name', 'item-code',
                                     'price', 'priceQty', 'effective-date']]

    extended_cal['start-date'] = extended_cal['start-date'].dt.strftime('%m/%d/%Y')
    extended_cal['end-date'] = extended_cal['end-date'].dt.strftime('%m/%d/%Y')
    extended_cal = extended_cal.rename(columns = {'start-date':'week-start-date',
                                                  'end-date':'week-end-date',
                                                  'priceQty':'reg-qty',
                                                  'price':'reg-price'})
    extended_cal = extended_cal.dropna()

    if change == 'Yes':
        extended_cal = extended_cal.groupby(['item-name', 'item-code']).filter(lambda x: (x['reg-price'].round(2) != x['reg-price'].round(2).min()).any())
    if change == 'No':
        extended_cal = extended_cal.groupby(['item-name', 'item-code']).filter(lambda x: (x['reg-price'].round(2) == x['reg-price'].round(2).min()).all())

    return extended_cal, prod_names, loc_names, cal['start-date'].min().strftime('%m/%d/%Y') + ' - ' + cal['end-date'].max().strftime('%m/%d/%Y')

# =============================================================================
#
# =============================================================================

# Same as above but with promotions instead of price.

def promotion_api(product_name = None, product_id = None, product_level = None, item_list = None,
                  active = 'Y', group_name = None,
             location_name = None, location_id = None, location_level = None,
             cal_year = None, quarter = None, period = None, week = None, day = None,
             start_date = None, end_date = None, calendar_id = None, cal_type = 'W',
             promo_type = None, page_no = None, block_no = None, user_id = None):

    cal_year, quarter, period, week, day, start_date, end_date, calendar_id = cal_logic(cal_year, quarter, period, week, day, start_date, end_date, calendar_id)

    cal = cal_lookup(cal_year, quarter, period, week, day,
                     start_date, end_date, calendar_id, cal_type)

    if cal.empty:
        raise ValueError('timeframe')

    location_name = catch_empty_list(location_name)

    locations = parse_loc_request(location_name, location_id, location_level, active)

    if locations.empty:
        raise ValueError('location')

    loc_names = list(locations['location-name'].drop_duplicates())
    locs = {l:[int(x) for x in locations.loc[locations['location-level'] == l, 'location-id']] for l in locations['location-level'].unique()}

    product_name = catch_empty_list(product_name)

    products = parse_prod_request(product_name, product_id, product_level,
                               user_id, active, n_responses)

    if products.empty:
        raise ValueError('product')

    if (products['authorized'] == False).all():
        raise ValueError('authorization')

    products = products.loc[products['authorized']]

    prod_names = list(products['product-name'].drop_duplicates())
    prods = {p:[int(x) for x in products.loc[products['product-level'] == p, 'product-id']] for p in products['product-level'].unique()}

    try:
        response = call_api('/promotion/get-promotion-data', locs, prods, cal, active)
    except:
        raise ValueError('missing_data')

    response = response.rename(columns = {'calendarId':'calendar-id'})
    response = response.loc[response['invalid_data'] != 'Y']
    response['sale-price'] = np.where(response['promotionTypeId'] == 3, response['mustBuyPrice'], response['offerPrice'])
    response['sale-qty'] = np.where(response['promotionTypeId'] == 3, response['mustBuyPriceQty'], response['offerPriceQty'])
    response = response.loc[response['calendar-id'].isin(cal['calendar-id'])]

    response = response.rename(columns = {'startDate':'promo-start-date',
                                          'endDate':'promo-end-date',
                                          'calendarStartDate':'week-start-date',
                                          'promotionTypeName':'promo-type',
                                          'pageNumber':'page-no',
                                          'blockNumber':'block-no',
                                          'regPrice':'reg-price',
                                          'regPriceQty':'reg-qty'})

    response = response.loc[:, ['week-start-date', 'item-name', 'promo-type',
                                'reg-price', 'reg-qty', 'sale-price', 'sale-qty',
                                'promo-start-date', 'promo-end-date',
                                'page-no', 'block-no']].drop_duplicates()

    response['block-no'] = response['block-no'].replace(0, np.nan)
    response['page-no'] = response['page-no'].replace(0, np.nan)
    response = response.sort_values(by = ['item-name', 'promo-start-date', 'promo-type',
                                          'sale-price', 'sale-qty', 'page-no', 'block-no'])
    response = response.groupby(['item-name', 'week-start-date', 'promo-type',
                                 'sale-price', 'sale-qty', 'promo-start-date',
                                 'promo-end-date']).first().reset_index()
    response = response.sort_values(by = ['item-name', 'promo-start-date', 'promo-type',
                                          'sale-price', 'sale-qty', 'page-no', 'block-no'])

    prices = price_api(product_name = product_name, product_id = product_id, product_level = product_level,
                       item_list = item_list, active = active,
                       location_name = location_name, location_id = location_id, location_level = location_level,
                       cal_year = cal_year, quarter = quarter, period = period, week = week, day = day,
                       start_date = start_date, end_date = end_date, calendar_id = calendar_id, cal_type = cal_type,
                       user_id = user_id, change = 'A')[0]

    response = response.merge(prices.loc[:, ['week-start-date', 'item-name', 'reg-price', 'reg-qty']],
                              how = 'inner')

    response = response.loc[:, ['item-name', 'reg-price', 'reg-qty', 'sale-price', 'sale-qty',
                                'promo-type', 'promo-start-date', 'promo-end-date', 'page-no', 'block-no']]
    response = response.drop_duplicates()

    if promo_type is not None:
        if type(promo_type) == str:
            response = response.loc[response['promo-type'] == promo_type]
        if type(promo_type) == list:
            response = response.loc[response['promo-type'].isin(promo_type)]

    if page_no is not None:
        if type(page_no) in [int, float]:
            response = response.loc[response['page-no'] == page_no]
        if type(page_no) == list:
            response = response.loc[response['page-no'].isin([int(x) for x in page_no if type(x) in [int, float]])]

    if block_no is not None:
        if type(block_no) in [int, float]:
            response = response.loc[response['block-no'] == block_no]
        if type(block_no) == list:
            response = response.loc[response['block-no'].isin([int(x) for x in block_no if type(x) in [int, float]])]

    return response, prod_names, loc_names, cal['start-date'].min().strftime('%m/%d/%Y') + ' - ' + cal['end-date'].max().strftime('%m/%d/%Y')

# =============================================================================
#
# =============================================================================

# Similar to above but with movement/sales/margin. Major difference is it queries sales analysis instead of
# using presto-data-services.

def movement_api(product_name = None, product_id = None, product_level = None, item_list = None,
                 active = 'Y', group_name = None,
                 location_name = None, location_id = None, location_level = None,
                 product_detail_level = None, location_detail_level = None,
                 cal_year = None, quarter = None, period = None, week = None, day = None,
                 start_date = None, end_date = None, calendar_id = None, cal_type = 'W',
                 promoted = 'All', user_id = None):

    # group name specifices at what level the user wants data to be aggregated.
    # E.g. if they want Grocery sales at the category/period level, then group_name = ['Catgeory', 'Period']
    # will do the trick. Uses the list of synonyms to figure out what to do.
    if group_name:
        syns = synonyms.copy(deep = True)
        syns['match'] = False
        if type(group_name) == str:
            group_name = [group_name]
        for g in group_name:
            match = extractOne(g,
                               list(synonyms['name'].values),
                               scorer = QRatio,
                               score_cutoff = 75)
            if match:
                syns.loc[syns['name'] == match[0], 'match'] = True

        for i, row in syns.loc[syns['match']].iterrows():
            if row['type'] == 'cal':
                cal_type = row['name'][0]
            if row['type'] == 'prod':
                product_detail_level = int(row['level'])
                product_col_name = row['name']
            if row['type'] == 'loc':
                location_detail_level = int(row['level'])
                location_col_name = row['name']

    cal_year, quarter, period, week, day, start_date, end_date, calendar_id = cal_logic(cal_year, quarter, period, week, day, start_date, end_date, calendar_id)

    cal = cal_lookup(cal_year, quarter, period, week, day,
                     start_date, end_date, calendar_id, cal_type)

    if cal.empty:
        raise ValueError('timeframe')

    location_name = catch_empty_list(location_name)

    locations = parse_loc_request(location_name, location_id, location_level, active)

    if locations.empty:
        raise ValueError('location')

    if location_detail_level:
        temp = loc_hier(location_id = [int(x) for x in locations['location-id'].values],
                        location_level = locations['location-level'].max(),
                        detail_level = location_detail_level)
        col_match = extractOne(location_col_name, list(temp.columns))
        if col_match:
            col_match = col_match[0]
        locations = parse_loc_request(location_id = list(temp[col_match].values),
                                      location_level = location_detail_level,
                                      active = active)

    loc_names = list(locations['location-name'].drop_duplicates())
    locs = {l:[int(x) for x in locations.loc[locations['location-level'] == l, 'location-id']] for l in locations['location-level'].unique()}

    product_name = catch_empty_list(product_name)

    products = parse_prod_request(product_name, product_id, product_level,
                                  user_id, active, n_responses)

    if products.empty:
        raise ValueError('product')

    if (products['authorized'] == False).all():
        raise ValueError('authorization')

    products = products.loc[products['authorized']]

    if product_detail_level:
        temp = prod_hier(product_id = [int(x) for x in products['product-id'].values],
                         product_level = products['product-level'].max(),
                         detail_level = product_detail_level)
        col_match = extractOne(product_col_name, list(temp.select_dtypes(include=[np.number]).columns.values))
        if col_match:
            col_match = col_match[0]
        products = parse_prod_request(product_id = list(temp[col_match].values),
                                      product_level = product_detail_level,
                                      active = active)

    prod_names = list(products['product-name'].drop_duplicates())
    prods = {p:[int(x) for x in products.loc[products['product-level'] == p, 'product-id']] for p in products['product-level'].unique()}

    if len(products) >= 10 ** 3 or len(locations) >= 10 ** 3 or len(cal) >= 10 ** 3:
        raise ValueError('data_size')

    responses = []
    base_table_name = 'sales_aggr' + ('_daily' if cal_type == 'D' else '_weekly' if cal_type == 'W' else '')
    for l in locs:
        table_name = base_table_name + '_rollup' if l <= 4 else base_table_name
        for p in prods:
            if l != 6:
                query = """
                select
                    calendar_id,
                    location_id,
                    location_level_id,
                    product_id,
                    product_level_id,
                    tot_visit_cnt,
                    tot_movement,
                    reg_movement,
                    sale_movement,
                    tot_revenue,
                    reg_revenue,
                    sale_revenue,
                    tot_margin,
                    reg_margin,
                    sale_margin,
                    avg_order_size
                from {table_name}
                where calendar_id in ({calendar_list})
                and location_level_id = {location_level}
                and location_id in ({location_list})
                """.format(table_name = table_name,
                           location_level = l,
                           location_list = ', '.join([str(x) for x in locs[l]]),
                           calendar_list = ', '.join(str(x) for x in cal['calendar-id'].values))

                if p != 99:
                    query += """
                    and product_id in ({product_list})
                    and product_level_id = {product_level}
                    """.format(product_level = p,
                               product_list = ', '.join([str(x) for x in prods[p]]))
                else:
                    query += """
                    and product_id is null
                    and product_level_id is null
                    """

            else:
                query = """
                select
                    calendar_id,
                    price_zone_id location_id,
                    6 as location_level_id,
                    product_id,
                    product_level_id,
                    sum(tot_visit_cnt) tot_visit_cnt,
                    sum(tot_movement) tot_movement,
                    sum(reg_movement) reg_movement,
                    sum(sale_movement) sale_movement,
                    sum(tot_revenue) tot_revenue,
                    sum(reg_revenue) reg_revenue,
                    sum(sale_revenue) sale_revenue,
                    sum(tot_margin) tot_margin,
                    sum(reg_margin) reg_margin,
                    sum(sale_margin) sale_margin,
                    avg(avg_order_size) avg_order_size
                from {table_name} sa
                join (select comp_str_id, price_zone_id from competitor_store
                      where price_zone_id in ({location_list})) cs
                    on cs.comp_str_id = sa.location_id
                    and sa.location_level_id = 5
                where calendar_id in ({calendar_list})
                """.format(table_name = table_name,
                           location_list = ', '.join([str(x) for x in locs[l]]),
                           calendar_list = ', '.join(str(x) for x in cal['calendar-id'].values))

                if p != 99:
                    query += """
                    and product_id in ({product_list})
                    and product_level_id = {product_level}
                    """.format(product_level = p,
                               product_list = ', '.join([str(x) for x in prods[p]]))
                else:
                    query += """
                    and product_id is null
                    and product_level_id is null
                    """

                query += """
                group by
                    calendar_id,
                    price_zone_id,
                    product_id,
                    product_level_id
                """

            connection = cx_Oracle.connect(username, password, dbname)
            response = pd.read_sql(query, connection)
            connection.close()
            response.columns = [x.lower().replace('_', '-') for x in response.columns]
            if response.empty:
                continue
            responses.append(response)

    try:
        response = pd.concat(responses, ignore_index = True)
    except:
        raise ValueError('missing_data')

    response = response.rename(columns = {'location-level-id':'location-level',
                                          'product-level-id':'product-level',
                                          'tot-visit-cnt': 'visits'})
    if 99 in products['product-level'].values:
        response['product-id'] = response['product-id'].fillna(products.loc[products['product-level'] == 99, 'product-id'].iloc[0])
        response['product-level'] = response['product-level'].fillna(99)

    if promoted == 'Yes':
        response['movement'] = response['sale-movement']
        response['sales'] = response['sale-revenue']
        response['margin'] = response['sale-margin']
    elif promoted == 'No':
        response['movement'] = response['reg-movement']
        response['sales'] = response['reg-revenue']
        response['margin'] = response['reg-margin']
    else:
        response['movement'] = response['tot-movement']
        response['sales'] = response['tot-revenue']
        response['margin'] = response['tot-margin']

    response = response.merge(cal, how = 'left', on = 'calendar-id')
    response = response.merge(products, how = 'left', on = ['product-id', 'product-level'])
    response = response.merge(locations, how = 'left', on = ['location-id', 'location-level'])

    if (response['margin'] == 0).all():
        rate = np.random.normal(loc = 0.3, scale = 0.05, size = len(response))
        response['margin'] = (rate * response['sales']).round(2)

    response = response.loc[:, [x for x in ['cal-year', 'quarter-no', 'period-no', 'week-no', 'day-no',
                                'start-date', 'end-date'] if x in cal.columns] + ['location-name', 'product-name',
                                'movement', 'sales', 'margin', 'visits', 'avg-order-size']]
    response = response.sort_values([x for x in ['cal-year', 'quarter-no', 'period-no', 'week-no', 'day-no',
                                'start-date', 'end-date'] if x in cal.columns] + ['location-name', 'product-name'])
    response['end-date'] = response['end-date'].fillna(response['start-date']).dt.strftime('%m/%d/%Y')
    response['start-date'] = response['start-date'].dt.strftime('%m/%d/%Y')

    if n_responses > 1:
        response = response.groupby([x for x in ['cal-year', 'quarter-no', 'period-no', 'week-no', 'day-no',
                                    'start-date', 'end-date'] if x in cal.columns] + ['location-name', 'product-name'])
        response = response.agg({'movement':'sum', 'sales':'sum', 'margin':'sum',
                                 'visits':'sum', 'avg-order-size':'mean'}).reset_index()

    response['margin-rate'] = (response['margin'] / response['sales']).fillna(0)
    response['avg-selling-price'] = (response['sales'] / response['movement']).fillna(0)
    return response, prod_names, loc_names, cal['start-date'].min().strftime('%m/%d/%Y') + ' - ' + cal['end-date'].fillna(cal['start-date']).max().strftime('%m/%d/%Y')

# =============================================================================
#
# =============================================================================

# Originally, we thought we might have the LLM look up the product hierarchy itself, but we never got to it.
# Not currently in use.
def hier_api(product_name = None, product_id = None, product_level = None, detail_level = None,
             other_levels = [5, 4, 3, 2, 1.5, 1], active = 'Y'):

    return prod_hier(product_name = product_name, product_id = product_id, product_level = product_level,
                     detail_level = detail_level, other_levels = other_levels, active = active)

# =============================================================================
#
# =============================================================================

# LLM will call this when it wants to format the data for plotting a table or chart.

def plotting_api(data, plot_type = 'table', metric_cols = ['sales'],
                 product_col = 'product-name', location_col = 'location-name'):

    df = pd.DataFrame(data)

    if 'week-no' in df.columns:
        time_col = 'week-no'
    elif 'period-no' in df.columns:
        time_col = 'period-no'
    elif 'quarter-no' in df.columns:
        time_col = 'quarter-no'
    else:
        time_col = 'cal-year'

    time_cols = ['cal-year', time_col] if time_col != 'cal-year' else ['cal-year']

    if isinstance(metric_cols, str):
        metric_cols = [word.strip() for word in metric_cols.split(',')]

    id_cols = time_cols + [product_col] + [location_col]
    all_cols = time_cols + [product_col] + [location_col] + metric_cols

    df = df[all_cols]
    df = df.drop_duplicates(subset=id_cols)

    unique_times = [int(x) for x in df[time_cols].drop_duplicates().loc[:, time_col].values]
    products = sorted(df[product_col].unique())
    locations = sorted(df[location_col].unique())

    if plot_type == 'line':
        options = { 'xaxis' : { time_col : unique_times}}

        if len(products) > 1 and  len(locations) == len(metric_cols) == 1:
            df_wide = df.pivot(index = time_cols, columns = product_col, values = metric_cols).fillna(0).round(2)
            cols = [f'{col[1]}' for col in df_wide.columns]
            df_wide.columns = cols

        elif len(locations) > 1 and  len(products) == len(metric_cols) == 1:
            df_wide = df.pivot(index = time_cols, columns = location_col, values = metric_cols).fillna(0).round(2)
            cols = [f'{col[1]}' for col in df_wide.columns]
            df_wide.columns = cols

        elif len(metric_cols) >= 1 and len(products) == len(locations) == 1:
            df_wide = df[metric_cols].fillna(0).round(2)
            cols = df_wide.columns

        else:
            df_wide = df.groupby(time_cols)[metric_cols].sum().fillna(0).round(2)
            cols = df_wide.columns

        series = [{'name': col, 'data': df_wide[col].tolist()} for col in cols]
        res = {'type': plot_type,  'options' : options, 'series' : series}

    elif plot_type == 'bar':

        if len(products) > 1:
            options = { 'xaxis' : { product_col : products}}
            df_wide = df.groupby(product_col)[metric_cols].sum().fillna(0).round(2)
            cols = df_wide.columns

        elif len(locations) > 1:
            options = { 'xaxis' : { location_col : locations}}
            df_wide = df.groupby(location_col)[metric_cols].sum().fillna(0).round(2)
            cols = df_wide.columns

        else:
            options = { 'xaxis' : { time_col : unique_times}}
            df_wide = df[metric_cols].fillna(0).round(2)
            cols = df_wide.columns

        series = [{'name': col, 'data': df_wide[col].tolist()} for col in cols]
        res = {'type': plot_type,  'options' : options, 'series' : series}

    elif plot_type == 'table':
        df.columns = [x.replace('-', ' ').title() for x in df.columns]
        tableData1 = df.fillna(0).round(2).to_dict(orient = 'records')
        res = {'type': plot_type,  'tableData1' : tableData1}

    else:
        raise ValueError('chart_type')

    return res


from lxml import etree
import pprint
import re
import xlrd
import csv
from utils import replace
import difflib

wb = xlrd.open_workbook('data/13.xls')
ws = wb.sheet_by_index(0)

def get_beats():
    for i in range(2, ws.nrows):
        __item = ws.row_values(i)[0:5]
        item = list(map(lambda x: str(x).strip(), __item))
        yield item

re_replace = re.compile(r'
')

def fetch(filename: str):
    with open(filename, 'r') as fr:
        return fr.read()

def fetch_info(cont: str) -> (bool, tuple):
    cont = re_replace.sub('
', cont)
    ele = etree.HTML(cont)
    result = []
    for tr_ele in ele.xpath('//*[@id="id12121xxx"]/table/tbody/tr'):
        code = tr_ele.xpath('td[2]/span[1]/text()')
        jar_code = tr_ele.xpath('td[2]/span[2]/text()')
        song = tr_ele.xpath('td[3]/strong/a/span/text()')
        lyrics = tr_ele.xpath('td[4]/table/tbody/tr[1]/td[2]/span/text()')
        song_singer = tr_ele.xpath('td[4]/table/tbody/tr[2]/td[2]/span/text()')
        singer = tr_ele.xpath('td[5]/span/text()')
        if len(jar_code) <= 0:
            jar_code = ['']
        result.append((
            code[0],
            str(jar_code[0]).replace('(', '').replace(')', ''),
            song[0],
            '',
            singer[0],
            lyrics[0],
            song_singer[0],
        ))
    return result

if __name__ == '__main__':
    data = []
    with open('data/test.csv', 'w', newline='') as csv_file:
        spam_writer = csv.writer(csv_file, delimiter='\t',
                                 quotechar='|', quoting=csv.QUOTE_MINIMAL)
        __header = [
            '伴奏ID', '后台歌手', '后台歌名', '大J歌名', 'JASRAC work code',
            'NexTone编码（上方）', 'J编码（下方）', '歌曲名',
            '副标题', '歌手', '作词', '作曲', '匹配百分比 %',
        ]
        ll = len(__header)
        spam_writer.writerow(__header)
        for i in get_beats():
            beat, singer, song, da_j, jar_code = i
            try:
                content = fetch('data/result/%s.html' % beat)
                items = fetch_info(content)
                if len(items):
                    percent = difflib.SequenceMatcher(None, replace(song), replace(items[0][2])).quick_ratio()
                    # for item in items:
                    spam_writer.writerow(i + list(items[0]) + ['%.2f' % (percent * 100), ])
                else:
                    spam_writer.writerow(i)
            except Exception as e:
                # print(e)
                spam_writer.writerow(i)
                pass
                # print(e)


import pickle
import numpy as np
import torch
from torch import nn
from torch.autograd import Variable
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from keras.utils import to_categorical
from torch.functional import Tensor
import torch.nn.functional as F

from flask import Flask, render_template,request
app = Flask(__name__)

class Model(nn.Module):
        def __init__(self, input_dim):
            super(Model, self).__init__()
            self.layer1 = nn.Linear(input_dim,50)
            self.layer2 = nn.Linear(50, 20)
            self.layer3 = nn.Linear(20, 3)

        def forward(self, x):
            x = F.relu(self.layer1(x))
            x = F.relu(self.layer2(x))
            x = F.softmax(self.layer3(x)) # To check with the loss function
            return x
'''
features, labels = load_iris(return_X_y=True)
features_train,features_test, labels_train, labels_test = train_test_split(features, labels, random_state=42, shuffle=True)

Training
model = Model(features_train.shape[1])
optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
loss_fn = nn.CrossEntropyLoss()
epochs = 100
x_train, y_train = Variable(torch.from_numpy(features_train)).float(), Variable(torch.from_numpy(labels_train)).long()
for epoch in range(1, epochs+1):
    # print ("Epoch #",epoch)
    y_pred = model(x_train)
    loss = loss_fn(y_pred, y_train)
    # print_(loss.item())

    # Zero gradients
    optimizer.zero_grad()
    loss.backward() # Gradients
    optimizer.step() # Update

predictions
x_test = Variable(torch.from_numpy(features_test)).float()
pred = model(x_test)
pred = pred.detach().numpy()

saving the model
torch.save(model, "swap.pkl")
'''
@app.route('/',methods=['GET', 'POST'])
def index():
    if request.get_json() is not None:
        content=request.get_json(silent=True)
        sepal_length =content['sepal_length']
        sepal_width =content['sepal_width']
        petal_length =content['petal_length']
        petal_width =content['petal_width']

    elif request.method=='POST':
        sepal_length=float(request.form['sl'])
        sepal_width=float(request.form['sw'])
        petal_length=float(request.form['pl'])
        petal_width=float(request.form['pw'])

    else:
        return render_template('index.html')

    saved_model = torch.load("swap.pkl")
    row=Tensor([sepal_length,sepal_width,petal_length,petal_width])
    n=np.argmax(saved_model(row).detach().numpy(), axis=0)
    columns=['Setosa','Versicolour','Virginica']
    return {"class":columns[n]}
if __name__ == '__main__':
    app.run(host='127.0.0.1', port=8000, debug=True)


# !/usr/bin/python3
# -*- coding: utf-8 -*-
"""
@Author         :  Zed
@Version        :  V1.0.0
------------------------------------
@File           :  22_guba_spider.py
@Description    :
@CreateTime     :  2020-4-28 13:46
------------------------------------
@ModifyTime     :
"""
import requests, re, json, time

def get_content(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4121.0 Safari/537.36 Edg/84.0.495.0',

    }
    response = requests.get(url, headers=headers)
    time.sleep(3)
    if response.status_code == 200:
        return response.text

def parse_html(html_str):
    balist_p = re.compile(r'
(.*?)
', re.S)
    balist = balist_p.search(html_str).group()
    li_p = re.compile(r'
(.*?)
', re.S)
    li_list = li_p.findall(balist)
    for li in li_list:
        # 阅读量
        read_p = re.compile(r'(.*?)', re.S)
        read = read_p.search(li).group(1).strip()
        # 评论
        comment_p = re.compile(r'.*(.*?)', re.S)
        comment = comment_p.search(li).group(1).strip()
        # 标题
        title_p = re.compile(r'class="note">(.*?)', re.S)
        title = title_p.search(li).group(1).strip()
        # 作者
        author_p = re.compile(r'target="_blank">(.*?)', re.S)
        author = author_p.search(li).group(1).strip()
        # 时间
        date_p = re.compile(r'(.*?)', re.S)
        date = date_p.search(li).group(1).strip()
        item = {}
        item['read'] = read
        item['comment'] = comment
        item['title'] = title
        item['author'] = author
        item['date'] = date
        guba_list.append(item)

def write_to_json(item):
    with open('guba_spider_infos.json', 'w', encoding='utf-8') as fp:
        json.dump(item, fp)
    print('写入成功')

def main():
    base_url = 'http://guba.eastmoney.com/default,99_%s.html'
    for i in range(1, 13):
        html_str = get_content(base_url % i)
        parse_html(html_str)
    write_to_json(guba_list)

if __name__ == '__main__':
    guba_list = []
    main()


from model.capsnet_model import CapsNet
from model.cnn_baseline import CNNBaseline

import tensorflow as tf
import numpy as np
import os
import sys

import logging
import daiquiri

from dataset import get_batch
from util import plot_imgs

daiquiri.setup(level=logging.DEBUG)
logger = daiquiri.getLogger(__name__)

class Net(object):

    def __init__(self, flags, hps):

        self.num_classes = flags.n_classes
        self.img_row = flags.n_img_row
        self.img_col = flags.n_img_col
        self.img_channels = flags.n_img_channels
        self.num_batch = flags.n_batch
        self.load_model_path = flags.load_model_path
        self.model_name = flags.model

        tf.reset_default_graph()
        g = tf.Graph()
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        config.gpu_options.per_process_gpu_memory_fraction = 0.33
        self.sess = tf.Session(config=config, graph=g)

        with g.as_default():
            tf.set_random_seed(1234)
            self.imgs = tf.placeholder(tf.float32,
                                       shape=[self.num_batch if flags.MODE == 'train' else None,
                                              self.img_row, self.img_col, self.img_channels])
            self.labels = tf.placeholder(tf.float32,
                                         shape=[self.num_batch if flags.MODE == 'train' else None,
                                                self.num_classes])
            models = {'cap': lambda: CapsNet(hps, self.imgs, self.labels),
                      'cnn': lambda: CNNBaseline(hps, self.imgs, self.labels)}
            self.model = models[flags.model]()
            logger.debug("Building Model...")

            self.model.build_graph()
            var_to_save = tf.trainable_variables() + [var for var in tf.global_variables()
                                                      if ('bn' in var.name) and ('Adam' not in var.name) and ('Momentum' not in var.name) or ('global_step' in var.name)]
            logger.debug(
                f'Building Model Complete...Total parameters: {self.model.total_parameters(var_list=var_to_save)}')

            self.summary = self.model.summaries
            self.train_writer = tf.summary.FileWriter("./train_log", self.sess.graph)
            self.test_writer = tf.summary.FileWriter("./test_log")
            self.saver = tf.train.Saver(var_list=var_to_save, max_to_keep=10)
            logger.debug(f'Build Summary & Saver complete')

            self.initialize()
            self.restore_model()

    def close(self):
        self.sess.close()
        logger.info(f'Network shutdown!')

    def initialize(self):
        init = (var.initializer for var in tf.global_variables())
        self.sess.run(list(init))
        logger.info('Done initializing variables')

    def restore_model(self, model_path=None):
        if self.load_model_path is not None:
            logger.debug('Loading Model...')
            try:
                ckpt = tf.train.get_checkpoint_state(check_point_path)
                self.saver.restore(self.sess, ckpt.model_checkpoint_path)
                logger.debug('Loading Model Succeeded...')
            except:
                logger.debug('Loading Model Failed')
        elif model_path is not None:
            try:
                self.saver.restore(self.sess, model_path)
                logger.debug('Loading Model Succeeded...')
            except:
                logger.debug('Loading Model Failed')
        else:
            logger.debug('No Model to load')

    def save_model(self, name):
        self.saver.save(self.sess, f'./savedmodels/model-{name}.ckpt',
                        global_step=self.sess.run(self.model.global_step))

    def reconstruct_img(self, features):
        assert self.model_name == "cap", "Assert model==cap in config.py"
        feed_dict = {self.model.masked_cigits: [features]}
        recon_imgs = self.sess.run([self.model.recon_images], feed_dict=feed_dict)
        plot_imgs(np.squeeze(recon_imgs[0], axis=(3,)))

    def train(self, porportion=0.25):
        logger.info('Train model...')
        num_iter = int(60000 * porportion // self.num_batch) + 1
        logger.info(f'1 Epoch training iteration will be: {num_iter}')
        for i in range(num_iter):
            batch = get_batch(train=True, batch_size=self.num_batch)

            feed_dict = {self.model.images: batch['x'],
                         self.model.labels: batch['y'],
                         self.model.is_training: True}
            try:
                _, summary, l, acc, lrn_rate = \
                    self.sess.run([self.model.train_op, self.summary, self.model.cost,
                                   self.model.acc, self.model.lrn_rate], feed_dict=feed_dict)
            except KeyboardInterrupt:
                self.close()
                sys.exit()
            except tf.errors.InvalidArgumentError:
                continue
            else:
                global_step = self.sess.run(self.model.global_step)
                self.train_writer.add_summary(summary, global_step)
                self.sess.run(self.model.increase_global_step)
                if i % 2 == 0:
                    logger.debug(
                        f'Train step {i} | Loss: {l:.3f} | Accuracy: {acc:.3f} | Global step: {global_step} | Learning rate: {lrn_rate}')

    def test(self, porportion=0.25):
        logger.info('Evaluate model...')
        num_iter = int(1000 * porportion // self.num_batch) + 1
        logger.info(f'1 Epoch training iteration will be: {num_iter}')
        t_l, t_acc = 0, 0
        for i in range(num_iter):
            batch = get_batch(train=False, batch_size=self.num_batch)

            feed_dict = {self.model.images: batch['x'],
                         self.model.labels: batch['y'],
                         self.model.is_training: False}
            try:
                summary, l, acc = \
                    self.sess.run([self.summary, self.model.cost,
                                   self.model.acc], feed_dict=feed_dict)
            except KeyboardInterrupt:
                self.close()
                sys.exit()
            except tf.errors.InvalidArgumentError:
                continue
            else:
                global_step = self.sess.run(self.model.global_step)
                self.test_writer.add_summary(summary, global_step)
                t_l += l
                t_acc += acc
                if i % 1 == 0:
                    logger.debug(
                        f'Test step {i} | Loss: {l:.3f} | Accuracy: {acc:.3f} | Global step: {global_step}')
        return t_l / num_iter, t_acc / num_iter


from __future__ import annotations

import asyncio
from contextvars import ContextVar
from dataclasses import dataclass, field
from functools import lru_cache
from typing import TYPE_CHECKING

from ._base import Middleware

if TYPE_CHECKING:
    import aiozipkin
    import opentelemetry.trace
    from datadog.dogstatsd import DogStatsd
    from prometheus_client import Counter, Histogram

    from .._context import Context, ErrorContext, OkContext

try:
    import prometheus_client
except ImportError:
    prometheus_client = None  # type: ignore[assignment]

try:
    import sentry_sdk
except ImportError:
    sentry_sdk = None  # type: ignore[assignment]

@dataclass(frozen=True)
class StatsdMiddleware(Middleware):
    """Emit statsd metrics using Datadog statsd client.

    We use Datadog statsd client because it is (compared to all alternatives)
    well maintained, type safe, and supports tags.

    Requires `datadog <https://github.com/DataDog/datadogpy>`_ package to be installed.
    """
    client: DogStatsd

    def on_start(self, ctx: Context) -> None:
        self.client.increment(
            f'walnats.{ctx.actor.event.name}.{ctx.actor.name}.started',
        )

    def on_failure(self, ctx: ErrorContext) -> None:
        self.client.increment(
            f'walnats.{ctx.actor.event.name}.{ctx.actor.name}.failed',
        )

    def on_success(self, ctx: OkContext) -> None:
        self.client.increment(
            f'walnats.{ctx.actor.event.name}.{ctx.actor.name}.processed',
        )
        self.client.histogram(
            f'walnats.{ctx.actor.event.name}.{ctx.actor.name}.duration',
            ctx.duration,
        )

@lru_cache(maxsize=256)
def _get_prometheus_counter(name: str, descr: str) -> Counter:
    if prometheus_client is None:
        raise ImportError('prometheus-client is not installed')
    return prometheus_client.Counter(
        name=name,
        documentation=descr,
        labelnames=['event', 'agent'],
    )

@lru_cache(maxsize=256)
def _get_prometheus_histogram(name: str, descr: str) -> Histogram:
    if prometheus_client is None:
        raise ImportError('prometheus-client is not installed')
    return prometheus_client.Histogram(
        name=name,
        documentation=descr,
        labelnames=['event', 'agent'],
    )

@dataclass(frozen=True)
class PrometheusMiddleware(Middleware):
    """Store Prometheus metrics.

    Requires `prometheus-client <https://github.com/prometheus/client_python>`_
    package to be installed.
    """

    def on_start(self, ctx: Context) -> None:
        _get_prometheus_counter(
            name='walnats_handler_started',
            descr='How many times handler was called',
        ).labels(ctx.actor.event.name, ctx.actor.name).inc()

    def on_failure(self, ctx: ErrorContext) -> None:
        _get_prometheus_counter(
            name='walnats_handler_failed',
            descr='How many times handler failed',
        ).labels(ctx.actor.event.name, ctx.actor.name).inc()

    def on_success(self, ctx: OkContext) -> None:
        _get_prometheus_counter(
            name='walnats_handler_succeeded',
            descr='How many messages handler processed',
        ).labels(ctx.actor.event.name, ctx.actor.name).inc()
        _get_prometheus_histogram(
            name='walnats_handler_duration',
            descr='How long it took for handler to process message',
        ).labels(ctx.actor.event.name, ctx.actor.name).observe(ctx.duration)

class SentryMiddleware(Middleware):
    """Report actor failures into Sentry using the official Sentry SDK.

    The failure report will include tags:

    * actor: actor name.
    * event: event name.

    Also, the "additional data" section will have:

    * delivered_at: timestamp when Nats delivered the message to the consumer.
    * stream_seq_id: sequence ID of the message in the Nats JetStream stream.

    Requires `sentry-sdk <https://github.com/getsentry/sentry-python>`_
    package to be installed.
    """
    __slots__ = ()

    def __init__(self) -> None:
        from sentry_sdk.integrations.logging import ignore_logger

        # disable capturing log message about actor failure
        ignore_logger('walnats.actor')

    # TODO(@orsinium): Can we have a task-local Sentry scope?
    # sentry_sdk.Hub is thread-local.
    # Example of making a scope:
    # https://github.com/jacobsvante/sentry-dramatiq/

    def on_failure(self, ctx: ErrorContext) -> None:
        # shouldn't fail because we import from sentry_sdk in __init__
        assert sentry_sdk is not None, 'sentry-sdk is not installed'
        sentry_sdk.capture_exception(
            error=ctx.exception,
            tags={
                'actor': ctx.actor.name,
                'event': ctx.actor.event.name,
            },
            extras={
                'stream_seq_id': ctx.seq_number,
                'delivered_at': str(ctx.metadata.timestamp),
            },
        )

@dataclass(frozen=True)
class ZipkinMiddleware(Middleware):
    """Emit Zipkin span.

    Requires `aiozipkin <https://github.com/aio-libs/aiozipkin>`_ package to be installed.
    """
    tracer: aiozipkin.Tracer
    sampled: bool | None = None
    _span: ContextVar[aiozipkin.SpanAbc | None] = field(
        default_factory=lambda: ContextVar('span', default=None),
        init=False,
    )

    @property
    def span(self) -> aiozipkin.SpanAbc | None:
        """The current span. Use it from handler to create child spans.

        May be ``None`` if you forgot to add the middleware for the actor,
        and so ``on_start`` hasn't been called.
        """
        return self._span.get()

    def on_start(self, ctx: Context) -> None:
        trace_ctx = self.tracer._next_context(sampled=self.sampled)
        if ctx.trace_id is not None:
            trace_ctx = trace_ctx._replace(trace_id=ctx.trace_id)
        span = self.tracer.to_span(trace_ctx)
        span.name(ctx.actor.name)
        span.tag('event', ctx.actor.event.name)
        span.kind('CONSUMER')
        span.start()
        self._span.set(span)

    def on_failure(self, ctx: ErrorContext) -> None:
        span = self._span.get()
        if span is not None:
            span.finish(exception=ctx.exception)  # type: ignore[arg-type]

    def on_success(self, ctx: OkContext) -> None:
        span = self._span.get()
        assert span is not None
        span.finish()

@dataclass(frozen=True)
class OpenTelemetryTraceMiddleware(Middleware):
    tracer: opentelemetry.trace.Tracer
    _span: ContextVar[opentelemetry.trace.Span | None] = field(
        default_factory=lambda: ContextVar('span', default=None),
        init=False,
    )

    @property
    def span(self) -> opentelemetry.trace.Span | None:
        """The current span. Use it from handler to create child spans.

        May be ``None`` if you forgot to add the middleware for the actor,
        and so ``on_start`` hasn't been called.
        """
        return self._span.get()

    def on_start(self, ctx: Context) -> None:
        import opentelemetry.trace
        from opentelemetry.trace.propagation import tracecontext

        trace_context: opentelemetry.trace.Context | None = None
        if ctx.trace_id is not None:
            trace_context = tracecontext.TraceContextTextMapPropagator().extract(
                carrier={'traceparent': ctx.trace_id},
            )
        span = self.tracer.start_span(
            name=ctx.actor.name,
            kind=opentelemetry.trace.SpanKind.CONSUMER,
            attributes={'event': ctx.actor.event.name},
            context=trace_context,
        )
        self._span.set(span)

    def on_failure(self, ctx: ErrorContext) -> None:
        span = self._span.get()
        if span is not None:
            if not isinstance(ctx.exception, asyncio.CancelledError):
                span.record_exception(ctx.exception)
            span.end()

    def on_success(self, ctx: OkContext) -> None:
        span = self._span.get()
        assert span is not None
        span.end()


#!/usr/bin/env python3
import random
from typing import Dict, List, Tuple
import rclpy
from rclpy.node import Node
from functools import partial

from turtlesim.srv import Kill
from turtlesim.srv import Spawn
from my_robot_interfaces.msg import NewTurtle
from my_robot_interfaces.msg import NewTurtleList
from my_robot_interfaces.msg import KillTurtle

class TurtleSpawner(Node):

    def __init__(self) -> None:
        super().__init__("turtle_spawner")
        self.declare_parameter("publish_frequency", 0.5)
        self.timer = self.create_timer((1 / self.get_parameter("publish_frequency").value), self.spawn_new_turtle_randomly)
        self.publish_timer = self.create_timer(0.5, self.publish_turtles)
        self.publisher = self.create_publisher(NewTurtleList, "new_turtles", 10)
        self.death_request_subscriber = self.create_subscription(KillTurtle, "kill_turtle", self.death_request_callback, 10)
        self.get_logger().info("Turtle Spawner started")
        self.new_turtle_number: int = 2
        self.living_turtles: Dict[str: Tuple(float, float)] = {}

    def spawn_new_turtle_randomly(self):
        x: float = random.uniform(1, 10)
        y: float = random.uniform(1, 10)
        theta: float = float(random.randint(0, 360))
        name: str = f"turtle{self.new_turtle_number}"
        self.get_logger().info(f"name: {name} x: {x} y: {y} theta: {theta}")
        self.call_spawn_new_turtle(x, y, theta, name)

    def call_spawn_new_turtle(self, x: float, y: float, theta: float, name: str):
        client = self.create_client(Spawn, "spawn")
        while not client.wait_for_service(timeout_sec=1.0):
            self.get_logger().warning("Waiting for turtle server...")
        request = Spawn.Request()
        request.x = x
        request.y = y
        request.theta = theta
        request.name = name
        future = client.call_async(request)
        future.add_done_callback(partial(self.callback_call_spawn_new_turtle, x=x, y=y, theta=theta, name=name))

    def callback_call_spawn_new_turtle(self, future, x: float, y: float, theta: float, name: str):
        try:
            response = future.result()
            self.get_logger().info(f"Turtle name: {response.name}")
            self.living_turtles[name] = (x, y)
            self.new_turtle_number += 1
        except Exception as e:
            self.get_logger().error(f"Service call failed {e}")

    def death_request_callback(self, msg):
        self.call_kill_turtle(turtle_name=msg.name)

    def call_kill_turtle(self, turtle_name: str):
        self.get_logger().info(f"Killing '{turtle_name}'")
        client = self.create_client(Kill, "kill")
        while not client.wait_for_service(timeout_sec=0.1):
            self.get_logger().warning("Waiting for Turtle...")

        request = Kill.Request()
        request.name = turtle_name

        future = client.call_async(request)
        future.add_done_callback(partial(self.callback_kill_turtle, name=turtle_name))

    def callback_kill_turtle(self, future, name: str):
        try:
            response = future.result()
            self.living_turtles.pop(name)
            self.get_logger().info(f"Turtle '{name}' killed")
        except Exception as e:
            self.get_logger().error(f"Service call failed {e}")

    # def publish_location(self, name: str, x: float, y: float):
    #     msg = NewTurtle()
    #     msg.x = x
    #     msg.y = y
    #     msg.name = name
    #     self.publisher.publish(msg)
    #     self.get_logger().info(f"New turtle {name} published")

    def publish_turtles(self):
        msg: NewTurtleList = NewTurtleList()
        turtles: List[NewTurtle] = []
        if self.living_turtles:
            for key, value in self.living_turtles.items():
                new_turtle: NewTurtle = NewTurtle()
                new_turtle.name = key
                new_turtle.x = value[0]
                new_turtle.y = value[1]
                turtles.append(new_turtle)
        msg.new_turtle_array = turtles
        self.publisher.publish(msg)

def main(args=None):
    rclpy.init(args=args)
    node = TurtleSpawner()
    rclpy.spin(node)
    rclpy.shutdown()

if __name__ == "__main__":
    main()
3


#coding:utf-8
'''
Created on 2017年1月17日

@author: coldJune
'''
from bs4 import BeautifulSoup
import urllib.request
import re

url='https://github.com/coldJune'

def catchLink(url):
    '''
                提 取网页中的超链接
    '''
    links=[]
    html=urllib.request.urlopen(url)
    html=html.read()
    soup=BeautifulSoup(html,'lxml')
    a=soup.findAll('a')
    for link in a:
        links=a['href']
    return
if __name__=='__main__':
    catchLink(url)


#!/usr/bin/python3
"""
This script lists the first State object from database passed into program
"""

import sys
from model_state import Base, State
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

if __name__ == "__main__":
    engine = create_engine('mysql+mysqldb://{}:{}@localhost:3306/{}'
                           .format(sys.argv[1], sys.argv[2], sys.argv[3]))
    Base.metadata.create_all(engine)
    Session = sessionmaker(bind=engine)
    session = Session()

    state = session.query(State).first()

    if not state:
        print("Nothing")
    else:
        print("{}: {}".format(state.id, state.name))


from django.shortcuts import render
from AHPSPK.models import *
from AHPSPK.forms import *
from django.shortcuts import redirect
from django.contrib import messages
import numpy as np
# Create your views here.
def TambahLaptop(request):
    if request.POST:
        form = FormLaptop(request.POST)
        if form.is_valid():
            form.save()
            form = FormLaptop
            konteks = {
                'form':form,
            }
            return (redirect('index'))
    else:
        form = FormLaptop()
        konteks = {
            'form':form,
        }
        return render(request,'tambah_laptop.html',konteks)

def SHOWLaptop(request):
    laptops = Laptop.objects.all()

    konteks = {
        'laptop' : laptops,
    }
    return render(request, 'index.html', konteks)

def ubah_data(request, id_laptop):
    laptop = Laptop.objects.get(id = id_laptop)
    if request.POST:
        form = FormLaptop(request.POST,instance=laptop)
        if form.is_valid():
            form.save()
            messages.success(request,"Data Berhasil Diubah")
            return redirect('ubah',id_laptop=id_laptop)
    else:
        form = FormLaptop(instance=laptop)
        konteks = {
            'form':form,
            'laptop':laptop
        }
        return render(request,'ubah_data.html',konteks)

def hapus_data(request, id_laptop):
    laptop = Laptop.objects.filter(id=id_laptop)
    laptop.delete()
    return redirect('index')

def coba(request):
    l = Laptop.objects.all()
    Kriteria = kriteria_nilai.objects.all()
    laptop=[]
    for x in l:
        laptop.append([x.Laptop,x.Harga,x.RAM,x.Processor.Nilai,x.Storage,x.Berat])
    kriteria = [Kriteria[0].kharga,Kriteria[0].kram,
            Kriteria[0].kprocessor,Kriteria[0].kstorage,
            Kriteria[0].kberat,]
    """
    kriteria = [
        1, 3, 2, 3, 3
    ]"""
    nama_kriteria=[
        "Harga","RAM","Processor","Storage","Berat"
    ]
    jadi = np.zeros([len(kriteria), len(kriteria)])
    jadi2 = np.zeros([len(kriteria), len(kriteria)])

    for x in range(len(kriteria)):
        for y in range(len(kriteria)):
            n1 = kriteria[x]
            n2 = kriteria[y]
            if n1 > n2:
                if n1 - n2 == 2:
                    jadi[x][y] = round(1 / 5, 2)
                elif n1 - n2 == 1:
                    jadi[x][y] = round(1 / 3, 2)
                elif n1 - n2 == 3:
                    jadi[x][y] = round(1 / 7, 2)
                elif n1 - n2 == 4:
                    jadi[x][y] = round(1 / 9, 2)

            elif n1 < n2:
                if n2 - n1 == 2:
                    jadi[x][y] = 5
                elif n2 - n1 == 1:
                    jadi[x][y] = 3
                elif n2 - n1 == 3:
                    jadi[x][y] = 7
                elif n2 - n1 == 4:
                    jadi[x][y] = 9

            elif n1 == n2:
                jadi[x][y] = 1

    T1 = jadi.sum(axis=0)

    for x in range(len(jadi)):
        for y in range(len(jadi[x])):
            ang = jadi[x][y]
            jadi2[x][y] = ang / T1[y]
    T2 = jadi2.sum(axis=0)
    for x in range(len(T2)):
        T2[x]= round(T2[x])

    putar = np.rot90(jadi2, 45)
    rata = putar.sum(axis=0)
    for y in range(len(rata)):
        rata[y] = rata[y] / len(putar)
    putar_rata = rata.reshape(len(rata), 1)
    kali = jadi.dot(putar_rata)
    bagi = np.zeros(len(kali))
    for x in range(len(kali)):
        bagi[x] = kali[x] / putar_rata[x]
    putar_bagi = bagi.reshape(len(bagi), 1)

    t = (1 / len(kriteria)) * putar_bagi.sum(axis=0)
    if (t<len(kriteria)):
        CI = (len(kriteria)-t) / (len(kriteria) - 1)
    else:
        CI = (t - len(kriteria)) / (len(kriteria) - 1)
    CR = CI / 1.12

    list_harga = []
    list_ram = []
    list_processor = []
    list_storage = []
    list_berat = []

    for x in laptop:
        list_harga.append(x[1])
        list_ram.append(x[2])
        list_processor.append(x[3])
        list_storage.append(x[4])
        list_berat.append(x[5])
    npharga = np.array(list_harga).reshape(len(list_harga), 1)
    min_harga = npharga.min(axis=0)
    rharga = np.array([float(min_harga / x) for x in npharga]).reshape(len(list_harga), 1)
    trasio_harga = rharga.sum(axis=0)
    normalisasi_harga = np.array([float(x / trasio_harga) for x in rharga]).reshape(len(list_harga), 1)
    # print("MIN=", min_harga)
    # print(rharga)
    # print(trasio_harga)

    npram = np.array(list_ram).reshape(len(list_ram), 1)
    total_ram = npram.sum(axis=0)
    # print(total_ram)
    normalisasi_ram = np.array([float(x / total_ram) for x in npram]).reshape(len(list_ram), 1)

    npprocessor = np.array(list_processor).reshape(len(list_processor), 1)
    total_processor = npprocessor.sum(axis=0)
    normalisasi_processor = np.array([float(x / total_processor) for x in npprocessor]).reshape(len(list_processor), 1)

    npstorage = np.array(list_storage).reshape(len(list_storage), 1)
    total_storage = npstorage.sum(axis=0)
    normalisasi_storage = np.array([float(x / total_storage) for x in npstorage]).reshape(len(list_storage), 1)

    npberat = np.array(list_berat).reshape(len(list_berat), 1)
    min_berat = npberat.min(axis=0)
    rberat = np.array([float(min_berat / x) for x in npberat]).reshape(len(list_berat), 1)
    trasio_berat = rberat.sum(axis=0)
    normalisasi_berat = np.array([float(x / trasio_berat) for x in rberat]).reshape(len(list_berat), 1)
    # print("MIN=", min_harga)
    # print(rharga)
    # print(trasio_harga)

    hasil = []
    for x in range(len(laptop)):
        hasil.append([float(normalisasi_harga[x]), float(normalisasi_ram[x]), float(normalisasi_processor[x]),
                      float(normalisasi_storage[x]), float(normalisasi_berat[x])])
    nphasil = np.array(hasil)
    rank = nphasil.dot(putar_rata)
    maxrank = rank.max(axis=0)

    hasil_akhir = []
    for x in range(len(laptop)):
        hasil_akhir.append(
            [laptop[x][0], float(normalisasi_harga[x]), float(normalisasi_ram[x]), float(normalisasi_processor[x]),
             float(normalisasi_storage[x]), float(normalisasi_berat[x]),float(rank[x])])

    hasil_akhir = sorted(hasil_akhir, key=lambda x: x[6], reverse=True)

    Tabel1_print =[]
    Tabel2_print = []
    for x in range(len(jadi)):
        Tabel1_print.append([nama_kriteria[x],jadi[x][0],jadi[x][1],jadi[x][2],jadi[x][3],jadi[x][4]])
        Tabel2_print.append([nama_kriteria[x], jadi2[x][0], jadi2[x][1], jadi2[x][2], jadi2[x][3], jadi2[x][4]])

    konteks={
        'laptop' : laptop,
        'jadi':Tabel1_print,
        'Tabel2':Tabel2_print,
        'T1':T1,
        'T2':T2,
        'ratarata':rata,
        'kriteria':nama_kriteria,
        't':t,
        'CI':CI,
        'CR':CR,
        'Hasilakhir':hasil_akhir,
    }

    return render(request,'hasil.html',konteks)

def testing(request):
    Kriteria = kriteria_nilai.objects.get(id=1)
    kriteria = [Kriteria.kharga, Kriteria.kram,
                Kriteria.kprocessor, Kriteria.kstorage,
                Kriteria.kberat, ]

    if request.POST:
        form = FormKriteria(request.POST, instance=Kriteria)
        if form.is_valid():
            form.save()
            return redirect('hasil')
    else:
        Kriteria = kriteria_nilai.objects.get(id=1)
        form = FormKriteria(instance=Kriteria)
        konteks = {
            'form': form,
            'kharga' : kriteria[0],
            'kram' : kriteria[1],
            'kprocessor' : kriteria[2],
            'kstorage' : kriteria[3],
            'kberat' :kriteria[4],
        }
        return render(request, 'testing.html', konteks)


import cv2
import os

def get_label_paths(path):
    path = os.path.join(path, 'labels')
    labels = []
    for filename in os.listdir(path):
        f = os.path.join(path, filename)
        # checking if it is a file
        if os.path.isfile(f):
            labels.append(f)

    return labels

def get_image_paths(path):

    path = os.path.join(path, 'images')
    images = []
    for filename in os.listdir(path):
        f = os.path.join(path, filename)
        # checking if it is a file
        if os.path.isfile(f):
            images.append(f)

    return images

def get_label(image, labels):
    for label in labels:
        label_name = str(label).replace('.txt', '').replace('labels', '')
        image_name = str(image).replace('.jpg', '').replace('images', '')

        if image_name == label_name:
            return label

    return None

def main():
    path_to_datasets_directory = ".."
    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), path_to_datasets_directory, 'datasets', 'supermarket_datasetV2')

    images = get_image_paths(path)
    labels = get_label_paths(path)

    with open(os.path.join(path, "classes.txt")) as f:
        classes = f.readlines()

    for image in images:
        label = get_label(image, labels)

        if label:
            visualized_image = cv2.imread(image)
            with open(label) as f:
                yolo_labels = f.readlines()
            for yolo_label in yolo_labels:
                yolo_label = [float(x) for x in yolo_label.split(' ')]
                product_class = int(yolo_label[0])

                yolo_label = yolo_label[1:]
                x1 = int(yolo_label[0]*visualized_image.shape[1] - 0.5*yolo_label[2]*visualized_image.shape[1])
                y1 = int(yolo_label[1]*visualized_image.shape[0] - 0.5*yolo_label[3]*visualized_image.shape[0])
                x2 = int(yolo_label[0]*visualized_image.shape[1] + 0.5*yolo_label[2]*visualized_image.shape[1])
                y2 = int(yolo_label[1]*visualized_image.shape[0] + 0.5*yolo_label[3]*visualized_image.shape[0])

                cv2.rectangle(visualized_image, (x1, y1), (x2, y2), (0,0,255), 2)

                cv2.putText(visualized_image, classes[product_class].replace('\n', ''), (int(x1), int(y1)), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (255, 255, 255), 2, cv2.LINE_AA)
            cv2.imshow('image', visualized_image)
            cv2.waitKey(0)

if __name__ == "__main__":
    main()


from django import forms
from .models import Day, Comment

class DayCreateForm(forms.ModelForm):

    class Meta:
        model = Day
        fields = '__all__'

class CommentCreateForm(forms.ModelForm):

    def __int__(self, *args, **kwargs):
        super().__int__(*args, **kwargs)
        for field in self.fields.values():
            field.widget.attrs['class'] = 'form-control'

    class Meta:
        model = Comment
        fields = ('name', 'text', 'date')


import asyncio
import logging

from aiogram.types import Message, ParseMode
from aiogram import Dispatcher

from main import dp
from service.main_menu import create_default_profile
from keyboards import replykeyboards, inlinekeyboards
from messages import default, templates

@dp.message_handler(commands=['start'])
async def send_welcome_messages(message: Message):
    """
        Sending welcome messages and set default user profile
    """
    user_id: int = message.from_user.id
    username: str = message.from_user.username
    user_full_name: str = message.from_user.full_name

    logging.info(f"User {user_id} enter in bot")

    task_creating_profile = asyncio.create_task(create_default_profile(user_id, username))
    await task_creating_profile
    await message.answer(templates.WELCOME_MESSAGE.format(user_full_name),
                         parse_mode=ParseMode.HTML, reply_markup=inlinekeyboards.TUTORIAL_MARKUP)
    await asyncio.sleep(1)
    await message.answer(default.MAIN_MENU_INFO_MESSAGE,
                         parse_mode=ParseMode.HTML,
                         reply_markup=replykeyboards.MAIN_MENU_MARKUP)

@dp.message_handler(lambda message: message.text == replykeyboards.BUTTON_RETURN_TO_MAIN_MENU)
async def return_to_main_menu(message: Message):
    """
        Load main menu if user press return button in another menu
    """
    await message.answer(default.MAIN_MENU_INFO_MESSAGE,
                         parse_mode=ParseMode.HTML,
                         reply_markup=replykeyboards.MAIN_MENU_MARKUP)

@dp.message_handler(commands=['menu'])
async def open_menu(message: Message):
    await return_to_main_menu(message)

@dp.message_handler(commands=['tutorial'])
async def send_tutorials(message: Message):
    await message.answer("https://t.me/seekleads/1")

@dp.message_handler(commands=['support'])
async def send_admin_contact(message: Message):
    await message.answer("При возникновении проблем при работе с ботом, напишите мне @Mr_GriM01",
                         reply_markup=inlinekeyboards.HELP_MARKUP)

@dp.message_handler(commands=['manager'])
async def send_manager_contact(message: Message):
    await message.answer("Контакт менеджера @I_III_I_II", reply_markup=inlinekeyboards.MANAGER_MARKUP)

@dp.message_handler(lambda message: message.text == replykeyboards.BUTTON_CHAT_LIST)
async def get_list_chat_settings(message: Message):
    """
        Load chats settings menu
    """
    await message.answer(default.CHAT_LIST_INFO,
                         parse_mode=ParseMode.HTML,
                         reply_markup=replykeyboards.CHAT_LIST_MARKUP)
    await asyncio.sleep(0.2)
    await message.answer(default.HELP_WITH_CHATS_MESSAGE,
                         reply_markup=inlinekeyboards.MANAGER_MARKUP)

@dp.message_handler(lambda message: message.text == replykeyboards.BUTTON_KEY_EXCEPT_WORDS)
async def get_key_except_words_settings(message: Message):
    """
        Load key/except words settings menu
    """
    await message.answer(default.KEY_EXCEPT_WORDS_INFO,
                         parse_mode=ParseMode.HTML,
                         reply_markup=replykeyboards.KEY_EXCEPT_MARKUP)

@dp.message_handler(lambda message: message.text == replykeyboards.BUTTON_SEARCHER_INSTANT)
async def get_instant_searcher_settings(message: Message):
    """
        Load searcher setting menu for instant result
    """
    await message.answer(default.SEARCHER_SETTINGS_INFO,
                         parse_mode=ParseMode.HTML,
                         reply_markup=replykeyboards.SEARCHER_INSTANT_MODE)

@dp.message_handler(lambda message: message.text == replykeyboards.BUTTON_INFO_SUBSCRIPTION)
async def get_subscription_info(message: Message):
    """
        Load subscription info menu
    """
    await message.answer(default.SUBSCRIPTION_SETTINGS_INFO,
                         reply_markup=replykeyboards.SUBSCRIPTION_INFO_MARKUP)

@dp.message_handler(lambda message: message.text == replykeyboards.BUTTON_SEARCHER_ONLINE)
async def get_online_searcher_settings(message: Message):
    """
        Load searcher setting menu for online search
    """
    await message.answer(default.ONLINE_SETTINGS_INFO,
                         reply_markup=replykeyboards.BOT_SETTINGS_MARKUP)

def register_main_menu(dispatcher: Dispatcher):
    dispatcher.register_message_handler(send_welcome_messages,
                                        commands=['start'])
    dispatcher.register_message_handler(open_menu,
                                        commands=['menu'])
    dispatcher.register_message_handler(send_tutorials,
                                        commands=['tutorial'])
    dispatcher.register_message_handler(send_admin_contact,
                                        commands=['support'])
    dispatcher.register_message_handler(send_manager_contact,
                                        commands=['manager'])
    dispatcher.register_message_handler(return_to_main_menu,
                                        lambda message: message.text == replykeyboards.BUTTON_RETURN_TO_MAIN_MENU)
    dispatcher.register_message_handler(get_list_chat_settings,
                                        lambda message: message.text == replykeyboards.BUTTON_CHAT_LIST)
    dispatcher.register_message_handler(get_key_except_words_settings,
                                        lambda message: message.text == replykeyboards.BUTTON_KEY_EXCEPT_WORDS)
    dispatcher.register_message_handler(get_instant_searcher_settings,
                                        lambda message: message.text == replykeyboards.BUTTON_SEARCHER_INSTANT)
    dispatcher.register_message_handler(get_subscription_info,
                                        lambda message: message.text == replykeyboards.BUTTON_INFO_SUBSCRIPTION)
    dispatcher.register_message_handler(get_online_searcher_settings,
                                        lambda message: message.text == replykeyboards.BUTTON_SEARCHER_ONLINE)


from queue import deque
from typing import List

def allPathsSourceTarget(start: int, end: int, graph: List[List[int]]) -> List[List[int]]:
        paths = []

        def backtrack(node, path):
            if node == end:
                paths.append(list(path))
                return
            for nextNode in graph[node]:
                path.append(nextNode)
                backtrack(nextNode, path)
                path.pop()

        path = deque([start])
        backtrack(start, path)
        return paths

g = [[4,3,1],[3,2,4],[3],[4],[]]

print(allPathsSourceTarget(1, 4, g))


import torch, logging
import torch.nn as nn
import numpy as np

from dsl import *
from pcfg import PCFG
from pcfg_logprob import LogProbPCFG

class Q_Predictor(nn.Module):
    def __init__(self,
        feature_extractor,
        template_dsl,
        template_cfg,
        list_variables):
        """
        feature_extractor: a neural network module
        taking a list of tasks (each task is a list of input-outputs)
        and returning a tensor of shape
        [len(list_of_tasks), feature_extractor.output_dimensionality]

        template_dsl: a dsl giving the list of primitives

        template_cfg: a cfg giving the structure that will be output
        """
        super(Q_Predictor, self).__init__()

        self.feature_extractor = feature_extractor
        H = self.feature_extractor.output_dimensionality # hidden

        self.template_cfg = template_cfg
        self.template_dsl = template_dsl

        self.number_of_primitives = len(template_dsl.list_primitives)
        self.list_variables = list_variables
        self.number_of_outputs = self.number_of_primitives + len(self.list_variables)
        self.number_of_parents = self.number_of_primitives + 1 # parent can be None
        self.maximum_arguments = max(len(primitive.type.arguments())
                                     for primitive in template_dsl.list_primitives)
        self.q_predictor = nn.Linear(H,
             self.number_of_outputs*self.number_of_parents*self.maximum_arguments)
        # print(self.number_of_outputs*self.number_of_parents*self.maximum_arguments)
    def q_vector_to_dictionary(self,q):
        """
        q: size self.number_of_outputs*self.number_of_parents*self.maximum_arguments
        """
        q = q.view(self.number_of_parents, self.maximum_arguments, self.number_of_outputs)
        q = nn.LogSoftmax(-1)(q)

        q_dictionary = {}
        for parent_index, parent in enumerate([None] + self.template_dsl.list_primitives):
            for argument_number in range(self.maximum_arguments):
                for primitive_index, primitive in enumerate(self.template_dsl.list_primitives):
                    q_dictionary[parent, argument_number, primitive] = \
                    q[parent_index, argument_number, primitive_index]
                for variable_index, variable in enumerate(self.list_variables):
                    q_dictionary[parent, argument_number, variable] = \
                    q[parent_index, argument_number, self.number_of_primitives + variable_index]
        return q_dictionary

    def forward(self, tasks):
        """
        tasks: list of tasks

        returns: list of PCFGs
        """
        features = self.feature_extractor(tasks)
        q = self.q_predictor(features)
        q_dictionary = [self.q_vector_to_dictionary(q[b])
              for b in range(len(tasks))]
        grammars = [self.template_cfg.Q_to_PCFG(q_dictionary[b])
              for b in range(len(tasks))]
        return grammars

    def train(self, programs, tasks, epochs=200):
        optimizer = torch.optim.Adam(self.parameters())

        for step in range(epochs):
            optimizer.zero_grad()
            grammars = self(tasks)
            likelihood = sum(grammar.log_probability_program(self.template_cfg.start, program)
                             for grammar,program in zip(grammars, programs))
            (-likelihood).backward()
            optimizer.step()

            if step % 100 == 0:
                logging.debug("optimization step {}\tlog likelihood {}".format(step, likelihood))
                logging.debug("grammars {}".format(grammars))

    def test(self, programs, tasks):
        grammars = self(tasks)
        for grammar, program in zip(grammars, programs):
            grammar = grammar.normalise()
            logging.debug("predicted grammar {}".format(grammar))
            logging.info("intended program {}\nprobability {}".format(program, grammar.probability_program(self.template_cfg.start, program)))


import json
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
from tmtoolkit.topicmod.evaluate import metric_coherence_gensim
import pandas as pd
import numpy as np
import pickle
from multiprocessing import Pool
from tqdm import tqdm
import multiprocessing as mp
import sys
import os

project_path = os.path.join(os.path.dirname(__file__), "..")
print(project_path)
num_cpus = mp.cpu_count()

def tokenize(doc):
    tokens = doc.split(" ")
    tokens = [word for word in tokens if len(word.strip()) > 0]
    return tokens

if __name__ == "__main__":

    print("Loading tech corpus")
    with open(os.path.join(project_path, "data/tech_review_word_corpus_nn_adj_adv.pkl"), "rb") as f:
        tech_review_corpus = pickle.load(f)
    reviews = pd.DataFrame(tech_review_corpus).review.tolist()

    print("Total workers:", num_cpus)

    print("Tokenize the corpus")

    with open(os.path.join(project_path, "data/stop_words.json"), "r") as f:
        stop_words = json.load(f)

    vectorizer = CountVectorizer(
        min_df=3, max_df=.90, tokenizer=tokenize, stop_words=stop_words, ngram_range=(1, 2))
    X = vectorizer.fit_transform(reviews)
    print("Total Vocab Size", len(vectorizer.vocabulary_))

    sum_words = X.sum(axis=0)
    words_freq = [(word, sum_words[0, idx])
                  for word, idx in vectorizer.vocabulary_.items()]
    print(sorted(words_freq, key=lambda x: x[1], reverse=True)[:50])

    topics_range = range(15, 6, -1)
    alpha = list(np.arange(0.01, 1, 0.3))
    alpha.append(None)
    beta = list(np.arange(0.01, 1, 0.3))
    beta.append(None)

    parameters = []
    for k in topics_range:
        for a in alpha:
            for b in beta:
                parameters.append({
                    "k": k, "alpha": a, "beta": b
                })

    print("Total parameter values to train", len(parameters))

    result = []

    for param in tqdm(parameters):
        aspect_file_name = "results/LDA/lda-aspect-k-{0}-a-{1}-b-{2}-reduced_words.json".format(
            str(param["k"])[:4],
            str(param["alpha"])[:4],
            str(param["beta"])[:4]
        )

        lda = LatentDirichletAllocation(
            learning_method="batch",
            random_state=100,
            n_components=param["k"],
            doc_topic_prior=param["alpha"],
            topic_word_prior=param["beta"],
            n_jobs=num_cpus - 2,
            max_iter=50
        )

        lda.fit(X)

        score = metric_coherence_gensim(measure='u_mass',
            top_n=25,
            topic_word_distrib=lda.components_,
            dtm=X,
            vocab=np.array([x for x in vectorizer.vocabulary_.keys()]),
            return_mean=True
        )

        result.append({
            "k":param["k"],
            "alpha":param["alpha"],
            "beta":param["beta"],
            "score":score
        })

        pd.DataFrame(result).to_csv(os.path.join(project_path,"data/lda_noun_umass.csv"))

    print("Done!")


# LeetCode:
# 16.02.2022
# Runtime: 410 ms, faster than 68.45% of Python3 online submissions for Find All Numbers Disappeared in an Array.
# Memory Usage: 26.2 MB, less than 10.15% of Python3 online submissions for Find All Numbers Disappeared in an Array.

# Test:
# if nums = [1,1]
# return [2]

from typing import List

def findDisappearedNumbers(nums: List[int]) -> List[int]:
    n_nums = [i for i in range(1, len(nums) + 1)]
    return [i for i in set(n_nums).difference(set(nums))]

nums = [4,3,2,7,8,2,3,1]
print(findDisappearedNumbers(nums))