slownik ={}

with open('oceny.txt') as oceny:
    for linia in oceny:
        linia = linia.strip() #ta funkcja usuwa wszelkie znaki, które nie są znakami - czyli bałe sacje (usuwa białe znaki z początku  ikońca linii)
        podzielone = linia.split() #przyjmuje argument po ktorym ma podzielic -- domyslny argument spacje
        nazwisko = podzielone[0]
        ocena = podzielone[1]
        ocena = int(ocena) #konwersja na int jeżeli string wyglada jak liczba
        slownik[nazwisko] = ocena

najwyzsza_ocena = 0
najwyzsza_ocena_nazwisko = None
najniznsza_ocena = 10000
najniznsza_ocena_nazwisko = None
suma = 0

for nazwisko in slownik:
    ocena = slownik[nazwisko]
    if ocena > najwyzsza_ocena:
        najwyzsza_ocena = ocena
        najwyzsza_ocena_nazwisko = nazwisko
    if ocena < najniznsza_ocena:
        najniznsza_ocena = ocena
        najniznsza_ocena_nazwisko = nazwisko
    suma += ocena #to jest dokadnie to samo co : suma = suma + ocena

print("najwyzsa ocena to: ", najwyzsza_ocena," uzyskana przez ", najwyzsza_ocena_nazwisko)
print("najwyzsa ocena to: ", najniznsza_ocena," uzyskana przez ", najniznsza_ocena_nazwisko)
print("Średnia: ", suma/len(slownik)) #len zwroci dlogosc na naszym slowniku czyli dlugosc

Courses/20181128 - Python basics/Zadanie1.py


#%%

from collections import defaultdict

import numpy as np
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestRegressor as RFR
from sklearn.datasets import load_boston
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

X, y = load_boston(return_X_y=True)

def rfrcv(target, train_data, param1, param2):
    val = cross_val_score(
        RFR(min_samples_split=param1,
            min_samples_leaf=param2,
            criterion='mse',
            random_state=2,
        ),
        train_data, target,
        cv=6, n_jobs=7
    ).mean()
    return val

def lerp(a, b, f):
    return a + f * (b - a)

params = {'min_samples_leaf': (0.01, 0.35),
         'min_samples_split': (0.01, 0.65),
        }

#%% exploring variation in hyperparameter over sample size
size_grid = list(range(100, len(X), 100))
# size_grid = [100, 200, 300, 400, 500]
steps = 10

grid_search = dict()
for size in size_grid:
    print(size, len(X))
    grid_search[size] = list()

    idx = np.random.randint(len(X), size=size)
    train_data = X[idx,:]
    target = y[idx]

    for n1 in range(steps):
        print(n1)
        param1 = round(lerp(params['min_samples_split'][0], params['min_samples_split'][1], n1/(steps-1)), 3)
        for n2 in range(steps):
            param2 = round(lerp(params['min_samples_leaf'][0], params['min_samples_leaf'][1], n2/(steps-1)), 3)
            val = rfrcv(target=target, train_data=train_data, param1=param1, param2=param2)
            grid_search[size].append([val, param1, param2])

# #%% marginal plots

# for size in size_grid:
#     d = grid_search[size]
#     score, p1, p2 = list(map(list, zip(*d)))

#     plt.scatter(p1, p2, c=score)
#     plt.title(str(size))
#     plt.show()

#%% 3d view is more appealing for the parameter space dimension
import matplotlib as mpl

colormap = mpl.cm.autumn
colorst = [colormap(40*i) for i in range(len(size_grid))]
title_font = {'fontname':'Arial', 'size':'26', 'color':'black', 'weight':'normal',
              'verticalalignment':'bottom'}

for i, size in enumerate(size_grid):
    d = grid_search[size]
    x, y, z = list(map(list, zip(*d)))

    x = np.array(x)
    y = np.array(y)
    z = np.array(z)

    for degree in range(60, 420, 360):
        print(i)
        ax = Axes3D(plt.figure(figsize=(15, 15)))
        ax.plot_trisurf(y, z, x, alpha=0.6, color=colorst[i])
        # ax.scatter(y, z, x, s=10, marker='o', c='red')
        ax.view_init(25, degree)
        # ax._axis3don = False
        # ax.w_zaxis.line.set_lw(0.)
        # ax.set_zticks([])
        ax.set_ylabel('min_samples_leaf')
        ax.set_xlabel('min_samples_split')
        ax.text2D(0.05, 0.45, "2D Text", transform=ax.transAxes)

        plt.title('Sample size: ' + str(round(size/5))+ '%', **title_font)
        plt.show()

'''
As in the one parameter case, the surface generated by the performance of the
model by the parameter tuple is pretty similar as we vary the sample size.
'''

#%% exploring variance in results from sample sizes

size_grid = list(range(100, len(X), 200))
# size_grid = [len(X)]
repeats = 10 # amount of models to estimate variation caused by different samples
steps = 10

variance = defaultdict(list)
for it in range(repeats):
    print(f'iteration: {it}')
    for size in size_grid:
        idx = np.random.randint(len(X), size=size)
        train_data = X[idx,:]
        target = y[idx]

        '''if we don't sample when the pct is 100, then there is no variance in
        the final result. Variation in the order of the data leads to different
        results even with the seed fixed on the algo side.'''
        # train_data = X
        # target = y
        for n1 in range(steps):

            param1 = round(lerp(params['min_samples_split'][0], params['min_samples_split'][1], n1/(steps-1)), 3)
            for n2 in range(steps):
                # print(round((n1 + n2)/steps**2,2))
                param2 = round(lerp(params['min_samples_leaf'][0], params['min_samples_leaf'][1], n2/(steps-1)), 3)
                val = rfrcv(target=target, train_data=train_data, param1=param1, param2=param2)
                variance[size].append([val, param1, param2])

for size in size_grid:
    d = variance[size]
    x, y, z = list(map(list, zip(*d)))

    x = np.array(x)
    y = np.array(y)
    z = np.array(z)

    for i in range(60, 420, 360):
        # print(i)
        ax = Axes3D(plt.figure(figsize=(15, 15)))
        ax.plot_trisurf(y, z, x, alpha=0.9)
        ax.scatter(y, z, x, s=10, marker='o', c='red', alpha=0.2)
        ax.view_init(25, i)
        # ax._axis3don = False
        ax.w_zaxis.line.set_lw(0.)
        ax.set_zticks([])
        plt.title(str(size))
        plt.show()

'''
There is more variance for smaller samples, so everything observed in the single
parameter case generalizes to a N-parameter case.
'''
#%% comparing mean/median of scores from smaller and biggest size. Do they match?

param_median = defaultdict(lambda: defaultdict(list))
param_mean = defaultdict(lambda: defaultdict(list))

for size in size_grid:
    p1_set = set(sorted([p1 for s, p1, p2 in variance[size]]))
    p2_set = set(sorted([p2 for s, p1, p2 in variance[size]]))
    for p1_ in p1_set:
        for p2_ in p2_set:
            s_lst = [s for s, p1, p2 in variance[size] if p1 == p1_ and p2 == p2_]
            param_median[p1_][p2_].append(round(np.median(s_lst),3))
            param_mean[p1_][p2_].append(round(np.mean(s_lst),3))
        # print(f'{size}: {p1}   {np.median(s_lst)}')

'''
It seems to hold
'''

#%% focusing on small samples distributions for all parameter values: gaussian or not?

'''

'''


def contains(ordered_list, number):
    # since the list is sorted
    # use binary search and find the number(or not) in O(log n)
    # time and O(1) space

    # initialize lower to zero
    lower = 0
    # initialize upper to length of ordered_list minus one
    upper = len(ordered_list) - 1

    # loop while lower <= upper
    while lower <= upper:
        # initialize midpoint to the average of lower and upper
        midpoint = (lower + upper) // 2

        # if number is equivalent to value at midpoint
        if number == ordered_list[midpoint]:
            # return True
            return True

        # if number is greater than value at midpoint
        if number > ordered_list[midpoint]:
            # set lower to midpoint plus one
            lower = midpoint + 1
        # otherwise
        else:
            # set upper to midpoint minus one
            upper = midpoint - 1

    # return False
    return False


from rest_framework.decorators import api_view
from django.shortcuts import render
from django.http import JsonResponse, HttpResponseRedirect

from .serializers import CandidateSerializer
from .forms import GradeForm
from .models import Candidate, Recruiter, Task, Grade

@api_view(['GET'])
def candidate_list(request):
   candidates = Candidate.objects.all()
   serializer = CandidateSerializer(candidates, many=True)

   return JsonResponse({'data': serializer.data}, json_dumps_params={'indent': 2})

@api_view(['GET', 'POST'])
def add_mark(request):
   my_form = GradeForm()

   context = {
      'form': my_form,
   }

   if request.method == 'POST':
      my_form = GradeForm(request.POST)

      if my_form.is_valid():
         grades = Grade.objects.filter(
            candidate__id=request.POST['candidate'],
            task__id=request.POST['task']
         )

         if grades.exists():
            return render(
               request,
               'appname/add_mark.html',
               {
                  'error': 'This task for this particular candidate was already grade!',
                  'success': ''
               }
            )
         else:
            Grade.objects.create(**my_form.cleaned_data)
            return render(
               request,
               'appname/add_mark.html',
               {
                  'error': '',
                  'success': 'Grade was added succesfully!'
               }
            )

      else:
         print(my_form.errors)

   return render(request, 'appname/add_mark.html', context)


from django.conf import settings

def main_context_processor(request):
    my_dict = {
        'website_address' : settings.WEBSITE_ADDRESS,
        'website_name_str' : settings.WEBSITE_NAME_STR,
        'website_name_slug': settings.WEBSITE_NAME_SLUG,
        'maintenance' : settings.WEBSITE_STATUS,
    }

    return my_dict

def emailing_context_preocessor(request):
    my_dict = {
        'emailer_team' : settings.EMAILER_TEAM
        }
    return my_dict


import math
import numpy as np

# *** GENERAL FUNCTIONS ***

def scale_coordinates(coordinates):
    scale_down_factor = []
    for i in range(2):

        # Max & min scaling
        c_max = max([c[i] for c in coordinates])
        c_min = min([c[i] for c in coordinates])

        c_diff = c_max - c_min

        # Scale each feature value
        for c in range(len(coordinates)):
            coordinates[c][i] = (coordinates[c][i] - c_min) / c_diff

        scale_down_factor.append(c_diff)

    return coordinates, scale_down_factor

# Euclidean distance when comparing Input Vectors
def euclidian_distance_input(i, j):
    return np.power(sum((i.get_feature_values()[iterator] - j.get_feature_values()[iterator]) ** 2 for iterator in range(len(i.get_feature_values()))), 0.5)

# Euclidean distance when comparing OutputNeurons
def euclidian_distance(i, j):
    return np.power(sum((i.weights[iterator] - j.weights[iterator]) ** 2 for iterator in range(len(i.weights))), 0.5)
    #return ((i.x - j.x) ** 2 + (i.y - j.y) ** 2) ** 0.5

def print_distances(distances):
    temp_string = ""
    for row in distances:
        temp_string += str(row[0]) + "\t" + row[1] + "\t" + row[2] + '\n'
    print(temp_string)

def file_reader(filename):
    with open(filename) as f:
        file = f.readlines()[5:]
        data = []
        for line in file:
            if line == 'EOF\n':
                break
            data.append(line.replace('\n', '').split())
    return data

def PointsInCircum(r, center_x, center_y, n=100):
        return [(math.cos(2*math.pi/n*x)*r+center_x, math.sin(2*math.pi/n*x)*r+center_y) for x in range(0, n+1)]

def argmin(list):
        argmin, minvalue = 0, list[0]
        for i,val in enumerate(list):
            if val < minvalue:
                argmin, minvalue = i, val
        return argmin, minvalue

def manhattan_distance(x,y):
    return sum([abs(x[i] - y[i]) for i in range(len(x))])

# def write_to_file(res_array):
#     import xlsxwriter
#     book = xlsxwriter.Workbook('module4results.xlsx')
#     sheet = book.add_worksheet("Results")
#
#     headers = ['Learning rate0', 'Learning rateTAU', 'Sigma0', 'SigmaTau']
#
#     for h in range(len(headers)):
#         sheet.write(0, h, headers[h])
#
#     for r in range(len(res_array)):
#         for i in range(len(res_array[r])):
#             sheet.write(r + 1, i, res_array[r][i])


#Leetcode 258
class Solution:
    def addDigits(self, num):
        """
        :type num: int
        :rtype: int
        """
        if(num < 10):
          return num

        sum = num
        while(1):
          str_num = str(sum)   #要遍历整数的各位，先str化，再list
          list_num = list(str_num)
          sum = 0
          for i in list_num:
            sum += int(i)

          if(sum < 10):
            return sum


#!/usr/bin/env python
# /home/atollye/current/programming_exercises/3_bars/bars.py

import os
import sys
import re
import json
import pprint
from geopy import distance

DEFAULT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                                                 "bars.json")
USER_LOCATION = (55.646354, 37.719613)

def get_path_to_data_file():
    message_1 = "\nПоскольку вы не указали путь к файлу с данными при"+ \
    " запуске скрипта, будет\nиспользован путь по умолчанию:\n{}\n"
    try:
        pth = sys.argv[1]
    except IndexError:
        pth = DEFAULT_PATH
        print(message_1.format(pth))
    return(pth)

def load_data(filepath):
    try:
        with open(filepath, "r") as file:
            file_obj = json.load(file)
    except json.JSONDecodeError:
        file_obj = None
    return file_obj

def check_data_structure(json_file):
    try:
        bar = json_file["features"][11]
        bar_name = bar["properties"]["Attributes"]["Name"]
        latitude = bar["geometry"]["coordinates"][1]
        longitude = bar["geometry"]["coordinates"][0]
        seats_count = bar["properties"]["Attributes"]["SeatsCount"]
        address = bar["properties"]["Attributes"]["Address"]
        phone = bar["properties"]["Attributes"]["PublicPhone"]
        correct = True
    except (KeyError, TypeError):
        correct = False
    return correct

def get_coordinates_from_user():
    print("\nВведите координаты интересующего вас места (в градусах "+
        "в формате десятичной \nдроби с точностью 6 знаков после точки)."+
        " \n\nПример:\nШирота 55.752805\nДолгота 37.622635\n\n")
    inpt = input("Введите широту:  ")
    lat = check_latitude(inpt)
    lon = None
    if lat:
        inpt = input("Введите долготу:  ")
        lon = check_longitude(inpt)
    coords = lat, lon
    return coords

def check_latitude(inpt):
    inpt = inpt.replace(",", ".")
    # 4 знака после запятой дают приемлимую точность определения координаты
    # на яндекс.каратах, 3 знака -- уже большое отклонение (около 70 метров):
    lat_regex = re.compile(r"(-)?\d\d\.\d\d\d\d(\d)?(\d)?(\d)?")
    match = re.search(lat_regex, inpt)
    if match:
        lat = float(match.group())
    else:
        lat = None
    # широта должна быть от −90° до +90°,
    if lat and (lat <=-90 or lat>=90):
        lat = None
    return lat

def check_longitude(inpt):
    inpt = inpt.replace(",", ".")
    lon_regex = re.compile(r"(-)?\d\d(\d)?\.\d\d\d\d(\d)?(\d)?(\d)?")
    match = re.search(lon_regex, inpt)
    if match:
        lon = float(match.group())
    else:
        lon = None
    #долгота должна быть от −180° до +180°
    if lon and (lon <=-180 or lon>=180):
        lon = None
    return lon

"""
    bar_name = bar["properties"]["Attributes"]["Name"]
    latitude = bar["geometry"]["coordinates"][1]
    longitude = bar["geometry"]["coordinates"][0]
    seats_count = bar["properties"]["Attributes"]["SeatsCount"]
"""

def get_biggest_bar(bars_dct):
    seats_lst = [bar["properties"]["Attributes"]["SeatsCount"] for
                                                           bar in bars_dct]
    max_seats = max(seats_lst)
    biggest= filter(lambda x: x["properties"]["Attributes"]["SeatsCount"] ==
    max_seats, bars_dct)

    #pprint.pprint(list(biggest))
    return biggest

def get_smallest_bar(bars_dct):
    seats_lst = [bar["properties"]["Attributes"]["SeatsCount"] for
                                                           bar in bars_dct]
    min_seats = min(seats_lst)
    smallest= filter(lambda x: x["properties"]["Attributes"]["SeatsCount"] ==
    min_seats, bars_dct)

    pprint.pprint(list(smallest))
    return smallest

def get_nearest_bar(user_coords, data):
    nearest = min(bars, key=lambda x: distance.distance(user_coords,
             (x["geometry"]["coordinates"][1],
              x["geometry"]["coordinates"][0])).km)
    pprint.pprint(nearest)
    return nearest

"""
Описание функции min-max
https://younglinux.info/python/feature/min-max
Известные алгоритмы на Python
https://younglinux.info/algorithm
"""

# def printout(results_list):
#     pass
#     #     print(""" "%s" (%s seats) \n""" % (bar, seats_count))

def error_exit(message=""):
    print(message)
    print("Попробуйте перезапустить скрипт и ввести данные заново")
    sys.exit()

def main():
    path_to_data = get_path_to_data_file()
    if not os.path.exists(path_to_data):
        error_exit("Файл с данными не существует")
    file_obj = load_data(path_to_data)
    if not file_obj:
        error_exit("Данные в файле не соответствуют формату json")
    if not check_data_structure(file_obj):
        error_exit("Данный json файл не содержит данные о барах" +
            " в требуемом программой формате")

    bars = file_obj["features"]
    pprint.pprint(get_biggest_bar(bars))

    # pprint.pprint(bars)
    # printout((get_biggest_bar(bars)))
    # printout(get_smallest_bar(bars))

    # coordinates = get_coordinates_from_user()
    # if not coordinates[0]:
    #     error_exit("\nВы ввели несуществующую широту\n")
    # elif not coordinates[1]:
    #     error_exit(("\nВы ввели несуществующую долготу\n"))

    # printout(get_nearest_bar(coordinates, bars_json))

if __name__ == '__main__':
    file_obj = load_data(DEFAULT_PATH)
    bars = file_obj["features"]
    get_nearest_bar(USER_LOCATION, bars)


"""
"""
ALPHA = 97

def lettersize(size):
    return 4*(size + 1) - 3

def dashsize(size, maxsize):
    return int((maxsize - lettersize(size))/2) - 2

def letterpart(size, maxsize):
    line = ""
    letter = ALPHA + maxsize - 1
    start = letter
    finish = letter - size
    while letter > finish:
        line += chr(letter)
        line += '-'
        letter -= 1
    line += chr(letter)
    letter += 1
    while letter <= start:
        line += '-'
        line += chr(letter)
        letter += 1
    return line

def print_rangoli(size):
    length = lettersize(size)

    for i in range(0, size):
        line = ""
        line += '-'*dashsize(i, length)
        line += letterpart(i, size)
        line += '-'*dashsize(i, length)
        print(line)

    for i in range(size - 2, -1, -1):
        line = ""
        line += '-'*dashsize(i, length)
        line += letterpart(i, size)
        line += '-'*dashsize(i, length)
        print(line)

if __name__ == '__main__':
    # n = int(input())
    print_rangoli(10)


"""Convert tsv into excel."""
import sys
import argparse
from argparse import ArgumentParser
import xlwt

style = xlwt.XFStyle()
font = xlwt.Font()
font.bold = True
style.font = font
borders = xlwt.Borders()
borders.bottom = xlwt.Borders.DASHED
style.borders = borders

def read_tsv(tsvfile):
    """Reads tsv file."""
    data = []
    with open(str(tsvfile), 'r') as tsv:
        for line in tsv:
            data.append([word.strip() for word in line.split("	")])
    return data

def write_xl1(master_list, outputname, outputdir):
    """Writes an excel file with content from master_list into separate sheets."""
    # print(master_list)
    wb = xlwt.Workbook()
    for j in range(len(master_list)):
        if 'Dummy' in master_list[j][0]:
            sheet = wb.add_sheet(str(master_list[j][0]))
            start_index = j
            # print(start_index)
            for i in range((start_index+1), len(master_list)):
                if len(master_list[i]) != 3:
                    end_index = i
                    temp_list = master_list[start_index:end_index]
                    # print(temp_list)
                    for row_index in range(len(temp_list)):
                        for col_index in range(len(temp_list[row_index])):
                            sheet.write(row_index, col_index, temp_list[row_index][col_index])
                    filename = str(outputdir) + '/' + str(outputname) + '.xls'
                    wb.save(filename)
                    break

def write_xl2(master_list, outputname, outputdir):
    """Writes an excel file with content from master_list into 1 sheet only."""
    # print(master_list)
    wb = xlwt.Workbook()
    sheet = wb.add_sheet(str(outputname))
    last_column = 0
    longest_column = 0
    filename = ''
    # j = 0
    for j in range(len(master_list)):
        if 'Du' in master_list[j][0][0:2]:
            if (j != len(master_list)-2) and ('Du' not in master_list[j+1][0][0:2]):
                start_index = j + 1
            else:
                start_index = j + 2
            for i in range(start_index, len(master_list)):
                # if len(master_list[i]) != 3:
                if (i == len(master_list)-1) or ('Dummy' in master_list[i][0]):
                    # separator for the experiments
                    end_index = i
                    if i != (len(master_list)-1):
                        temp_list = master_list[start_index:end_index]
                    else:
                        temp_list = master_list[start_index:end_index+1]
                    if 'Du' in temp_list[-1][0]:
                        temp_list.pop(len(temp_list)-1)
                    print(temp_list)
                    for row_index in range(len(temp_list)):
                        if len(temp_list) > longest_column:
                            longest_column = len(temp_list)
                        for col_index in range(len(temp_list[row_index])):
                            new_col = (last_column + col_index)
                            if len(temp_list[row_index]) != 1:
                                sheet.write(row_index, new_col, temp_list[row_index][col_index])
                            else:
                                sheet.write(row_index, new_col, temp_list[row_index][col_index])
                                sheet.write(row_index, new_col+1, '% Coverage', style=style)
                                # sheet.write(row_index, new_col+2, '% Blob', style=style)
                    print(len(temp_list[1]))
                    last_column += len(temp_list[1])
                    filename = str(outputdir) + '/' + str(outputname) + '.xls'
                    wb.save(filename)
                    break
    sheet.write(longest_column + 1, 0, 'Summary Stats')
    sheet.write(longest_column + 2, 0, 'Mean')
    sheet.write(longest_column + 3, 0, 'Median')
    sheet.write(longest_column + 4, 0, 'Min')
    sheet.write(longest_column + 5, 0, 'Max')
    sheet.write(longest_column + 6, 0, 'Standard Dev')
    wb.save(filename)

if __name__ == '__main__':
    def str2bool(v):
        """Make switches with default values but with correct parser error."""
        if v.lower() in ('yes', 'true', 't', 'y', '1'):
            return True
        elif v.lower() in ('no', 'false', 'f', 'n', '0'):
            return False
        else:
            raise argparse.ArgumentTypeError('Boolean value expected.')
    # parser.add_argument('--is_debug', default=False, type=lambda x: (str(x).lower() == 'true'))
    # parser = argparse.ArgumentParser(description='A tsv to excel converter for Bellas muscle fibres')
    # parser.add_argument('tsvfile', help='tsv file generated from fiji')
    # parser.add_argument('outputname', help='excel file name')
    # parser.add_argument('outputdir', help='excel output directory')
    # parser.add_argument('numbersheets', nargs='?', type=str2bool, const=True, default='False',
    #                     help='specify if you want separate sheets per experiment or not')
    # args = parser.parse_args()
    # master = read_tsv(args.tsvfile)
    # if args.numbersheets:
    #     print('1')
    #     write_xl1(master, args.outputname, args.outputdir)
    # else:
    #     print('2')
    #     write_xl2(master, args.outputname, args.outputdir)
    master = read_tsv(sys.argv[1])
    if sys.argv[4] is "True":
        print('1')
        write_xl1(master, sys.argv[2], sys.argv[3])
    else:
        print('2')
        write_xl2(master, sys.argv[2], sys.argv[3])


class FlightData:

    def __init__(self):
        self.from_city = ''
        self.to_city = ''
        self.travel_price = None
        self.travel_link = None
        self.format_travel = {
            'from_city': '',
            'to_city': '',
            'option': [],
            "lowest_option": {},
        }

    def format_travel_info(self, flight_json, city_data):
        """Receives flight raw data and city data and format with useful information"""
        self.clear_format_travel_dict()

        self.from_city = flight_json['data'][0]['cityFrom']
        self.to_city = flight_json['data'][0]['cityTo']
        self.format_travel['from_city'] = self.from_city
        self.format_travel['to_city'] = self.to_city

        lowest_price = city_data['lowestPrice']
        lowest_travel_link = None
        for flight in flight_json['data']:
            self.travel_price = flight['price']
            self.travel_link = flight['deep_link']
            if lowest_price > self.travel_price:
                lowest_price = self.travel_price
                lowest_travel_link = self.travel_link
            self.format_travel["option"] += [
                {
                    'travel price': self.travel_price,
                    'link': self.travel_link,
                }
            ]
        self.format_travel["lowest_option"] = {
            'travel price': lowest_price,
            'link': lowest_travel_link,
        }

    def clear_format_travel_dict(self):
        """Clear Dictionary and reset keys to default values"""
        self.format_travel.clear()
        self.format_travel = {
            'from_city': '',
            'to_city': '',
            'option': [],
            "lowest_option": {},
        }

Day 040 Project Flight Club/flight_data.py


import os
import rox
from rox import OptionsBox, g
import string

from rox.basedir import xdg_data_dirs

def build_theme_button(box, node, label):
    b = g.Button()
    b.set_label(label)
    box.may_add_tip(b, node)
    def open_theme_folder(self):
        from rox import filer
        filer.open_dir(os.path.join(os.path.expanduser("~"), '.icons'))
    #def get():
    #		return 'false'
    #def set():

    #box.handlers[option] = (get, set)
    b.connect('clicked', open_theme_folder)
    return [b]

def add_icon_themes(themes, dir):
    if not os.path.isdir(dir):
        return
    for theme in os.listdir(dir):
        if theme.startswith('.'): continue
        leaf = os.path.join(dir, theme)
        if os.path.isdir(os.path.join(leaf, 'cursors')) or \
           theme == "core_theme" and os.path.exists(os.path.join(leaf, 'index.theme')):
            themes[theme] = True

def build_icon_theme(box, node, label, option):
    hbox = g.HBox(False, 4)

    hbox.pack_start(g.Label(_(label)), False, True, 0)

    button = g.OptionMenu()
    hbox.pack_start(button, True, True, 0)

    menu = g.Menu()
    button.set_menu(menu)

    themes = {}
    add_icon_themes(themes, os.path.expanduser('~/.icons'))
    add_icon_themes(themes, '/usr/share/icons')
    add_icon_themes(themes, '/usr/share/pixmaps')
    add_icon_themes(themes, '/usr/X11R6/lib/X11/icons')
    add_icon_themes(themes, '/usr/X11/lib/X11/icons')

    names = themes.keys()
    names.sort()
    for name in names:
        if name == "core_theme":
            del names[names.index(name)]
            names.insert(0, _("Core Theme"))

    names.insert(0, _("No Theme"))

    for name in names:
        item = g.MenuItem(name)
        menu.append(item)
        item.show_all()

    def update_theme():
        i = -1
        for kid in menu.get_children():
            i += 1
            item = kid.child

            # The label actually moves from the menu!!
            if not item:
                item = button.child
            label = item.get_text()
            if label == option.value or label == _("No Theme"):
                button.set_history(i)

    def read_theme():
        return button.child.get_text()

    box.handlers[option] = (read_theme, update_theme)

    button.connect('changed', lambda w: box.check_widget(option))

    return [hbox]

OptionsBox.widget_registry['icon-theme'] = build_icon_theme
OptionsBox.widget_registry['folder-button'] = build_theme_button


# coding=utf-8
"""
Задание 5 *
Дано множество отрезков, представляющих собой рёбра дорожного графа (файл data.xls).

F_VerticeId — id первой вершины ребра
T_VerticeId - id второй вершины ребра
dist — вес ребра (расстояние)
F_POINT_X — долгота вершины F_VerticeId
F_POINT_Y - широта вершины F_VerticeId
T_POINT_X - долгота вершины T_VerticeId
T_POINT_Y - широта вершины T_VerticeId

На входе программа должна получать координаты двух точек, причём:
38,9774837690821 <  долгота < 39,0283642309179
45,0549145480683 < широта < 45,0908534519317
Эти точки необязательно являются вершинами графа. Они могут находиться вблизи дорог.

Необходимо построить кратчайший маршрут по дорогам между двумя точками, которые были даны на входе, и вывести расстояние
между этими точками по дорогам.
Визуализировать результат. То есть необходимо нанести на карту все дороги из файла data.xls, и полученный кратчайший
маршрут выделить каким-либо цветом, отличным от цвета других дорог.

"""
import math
import os
import simplekml
import xlrd

# края точек в data.xml
border = {'left': 38.7518678245754, 'right': 38.8036140188757, 'bottom': 45.0968821926186, 'top': 45.1335440675189}

# столбец матрицы ( матрица, индекс столбца)
def generatorCollumn(matrix, colnum):
    return [init[colnum] for init in matrix]

# чтение таблицы из файла
def dataInit(file):
    sheet = xlrd.open_workbook(file, formatting_info=True).sheet_by_index(0)

    # data = {}
    # for rownum in range(sheet.nrows):
    #     if rownum != 0:
    #         p = int(sheet.row_values(rownum)[0]), int(sheet.row_values(rownum)[1])
    #         data.update({p: [float(sheet.row_values(rownum)[2]),
    #                          float(sheet.row_values(rownum)[3]), float(sheet.row_values(rownum)[4]),
    #                          float(sheet.row_values(rownum)[5]), sheet.row_values(rownum)[6]]})
    # return data
    # try:
    #     list1 =float(sheet.row_values(4)[6].split())
    # except ValueError as e:
    #     print( "error", e)

    return tuple([int(sheet.row_values(rownum)[0]),
                  int(sheet.row_values(rownum)[1]),
                  float(sheet.row_values(rownum)[2]),
                  float(sheet.row_values(rownum)[3]),
                  float(sheet.row_values(rownum)[4]),
                  float(sheet.row_values(rownum)[5]),
                  float(sheet.row_values(rownum)[6])] for rownum in range(sheet.nrows) if rownum != 0)

    # кортеж исходной таблицы

# получение точек
def generatorPoints(data):
    gen = {}
    for road in data:
        if not (road[0] in gen):
            gen.update({int(road[0]): [road[2], road[3]]})
        if not (road[1] in gen):
            gen.update({int(road[1]): [road[4], road[5]]})
    return gen

# нахождение границ
def borderPoints(points):
    left = 180
    right = 0
    bottom = 90
    top = 0
    for point in points:
        left = mmm(left, points.get(point)[0])
        right = max(right, points.get(point)[0])
        bottom = mmm(bottom, points.get(point)[1])
        top = max(top, points.get(point)[1])
    return {'left': left, 'right': right, 'bottom': bottom, 'top': top}

# Ввод
def init():
    # проверка ввода
    def initCheck():
        while True:
            x = float(input())
            if border.get('left') <= x <= border.get('right') or border.get('bottom') <= x <= border.get('top'):
                return x
            else:
                print('Ошибка! Введте корректные данные!')

    print('Введите координаты первой точки: долгота широта')
    a = [initCheck(), initCheck()]

    print('Введите координаты второй точки')
    b = [initCheck(), initCheck()]
    return a, b

# поиск ближайшей точки
def findPoint(a, points):
    x = list(points.keys())[0]
    # print(x)
    a.append(x)
    distance = math.sqrt((points.get(x)[0] - a[0]) ** 2 + (points.get(x)[1] - a[1]) ** 2)
    for point in points:
        dist1 = math.sqrt((points.get(point)[0] - a[0]) ** 2 + (points.get(point)[1] - a[1]) ** 2)
        if distance > dist1:
            a[2] = point
            distance = dist1

# не работает должным образом
def floidBellman(point1, data):
    # m - количество ребер
    # n - количество вершин

    points = dict.fromkeys(generatorPoints(data).keys(), 200000)
    points.update({point1: 0})
    for point in points:
        if point != point1:
            for road in data:
                if points.get(road[1]) > (points.get(road[0]) + float(road[6])):
                    points.update({road[1]: points.get(road[0]) + float(road[6])})
                if points.get(road[0]) > points.get(road[1]) + float(road[6]):
                    points.update({road[0]: points.get(road[1]) + float(road[6])})

    return points

# соседние вершины, в которые идут ребра
def waysTo(data):
    ways = {}
    for way in data:
        if not ways.get(way[0]):
            ways.setdefault(way[0], [way[1]])
        else:
            c = list(ways.get(way[0]))
            c.append(way[1])
            ways.update({way[0]: c})
        # if not ways.get(way[1]):
        #     ways.setdefault(way[1], [way[0]])
        # else:
        #     c = list(ways.get(way[1]))
        #     c.append(way[0])
        #     ways.update({way[0]: c})
    return ways

# соседние вершины, из которых идут ребра
def waysFrom(data):
    ways = {}
    for way in data:
        if not ways.get(way[1]):
            ways.setdefault(way[1], [way[0]])
        else:
            c = list(ways.get(way[1]))
            c.append(way[0])
            ways.update({way[0]: c})
    return ways

def dijkstra(point1, point2, data):
    dist = dict.fromkeys(generatorPoints(data).keys(), 10000)

    # Соседние вершины
    wt = waysTo(data)
    dist.update({point1: 0})
    parents = dict.fromkeys(generatorPoints(data).keys(), False)
    mark = parents.copy()

    # пути
    roads = {(road[0], road[1]): road[6] for road in data}
    # roads.update({(road[1], road[0]): road[6] for road in data})
    v = point1
    if wt.get(v):
        for j in wt.get(v):
            now = dist.get(v)
            p = v, j
            to = now + roads.get(p)

            if dist.get(j) > to:
                dist.update({j: to})
                parents.update({v: j})
    for i in dist.keys():

        # выбирается вершина с наименьшей величиной dist.get(j)
        for j in dist.keys():
            if not (mark.get(j)) and (v == point1 or dist.get(j) < dist.get(v)):
                v = j
        # if dist.get(v) ==100:
        #     continue

        # вершина помечается
        mark.update({v: True})
        # просматриваем все ребра (v,to) и пытаемся улучшить значение dist.get(to)
        if wt.get(v):
            for j in wt.get(v):
                way=v,j
                newadd = roads.get(way)
                if newadd:
                    now = dist.get(v)
                    p = v, j

                    frome=dist.get(j)
                    to = dist.get(v) + roads.get(way)
                    if frome > dist.get(v) + roads.get(way):
                        dist.update({j: to})
                        parents.update({v: j})

    return dist

data = dataInit('data.xls')
# словарь точек
points = generatorPoints(data)

# a, b = init()
# a, b = [38.75337683, 45.11878547], [38.8036140188757, 45.0968821926186]
a, b = 51362963, 51986257
ways=waysTo(data)
while True:
    print(ways.get(a))
    roads = {(road[0], road[1]): road[6] for road in data}
    w=[(a,b) for b in ways.get(a)]
    print([roads.get(i) for i in w])
    a=int(input())

# findPoint(a, points)
# findPoint(b, points)

# roads = floidBellman(a[2], data)
roads = dijkstra(a, b, data)
mmm = 10000
# vl = (list(roads.values()))
# for i in vl:
#     if i != 0:
#         mmm = min(mmm, i)
print(roads.get(b))
# kml = simplekml.Kml()
# for road in data:
#     # kml.newpoint(coords=[(points.get(point))])
#     kml.newlinestring(description=str(road[6]),
#                       coords=[(road[2], road[3]), (road[4], road[5])]).style.linestyle.color = simplekml.Color.blue
#
# kml.save("5.kml")
"""
проблема в неверном условии:
38,9774837690821 <  долгота < 39,0283642309179
45,0549145480683 < широта < 45,0908534519317

    точки в файле
    """


import torch
import jieba
import gensim
from transformers import BertTokenizer, BertForMaskedLM
from scipy.special import softmax
import numpy as np
import traceback
import OpenHowNet

def substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, substitution_words, word_freq_dict, substitution_num):

    MAX = 56065

    loss_scores = []
    freq_scores = []
    sim_scores = []
    hownet_scores = []

    for i in range(len(substitution_words)):
        word = substitution_words[i]
        try:
            freq_scores.append(int(word_freq_dict[word]))
        except:
            freq_scores.append(0)
        sentence_splited = row_line.split('\t')[0].split(' ')
        assert source_word in sentence_splited
        sentence = cut_out(sentence_splited, source_word, 5)
        sub_sentence = sentence.replace(source_word, word)
        loss = sent_loss(model, tokenizer, sub_sentence)
        loss_scores.append(loss)
        try:
            similarity = model_word2vector.similarity(source_word, word)
            sim_scores.append(similarity)
        except:
            sim_scores.append(0)
        try:
            similarity = hownet.calculate_word_similarity(source_word, word)
            hownet_scores.append(similarity)
        except:
            hownet_scores.append(0)

    assert len(loss_scores) == len(freq_scores) == len(sim_scores) == len(hownet_scores)
    loss_scores_sorted = sorted(loss_scores)
    loss_ranks = [loss_scores_sorted.index(x) + 1 for x in loss_scores]
    freq_scores_sorted = sorted(freq_scores)
    freq_ranks = [freq_scores_sorted.index(x) + 1 for x in freq_scores]
    sim_scores_sorted = sorted(sim_scores, reverse=True)
    sim_ranks = [sim_scores_sorted.index(x) + 1 for x in sim_scores]
    hownet_scores_sorted = sorted(hownet_scores, reverse=True)
    hownet_ranks = [hownet_scores_sorted.index(x) + 1 for x in hownet_scores]
    # TODO: rank normalization
    all_ranks = [[substitution_word, loss+freq+sim+hownet] for substitution_word, loss, freq, sim, hownet in zip(substitution_words, loss_ranks, freq_ranks, sim_ranks, hownet_ranks)]
    ss_sorted = sorted(all_ranks, key=lambda x:x[1])
    ss_sorted = [x[0] for x in ss_sorted]
    freq_rank_source = int(word_freq_dict[source_word]) if source_word in word_freq_dict else MAX
    try:
        freq_rank_next = int(word_freq_dict[ss_sorted[1]])
    except:
        freq_rank_next = MAX - 1
    if ss_sorted[0] == source_word and freq_rank_source > freq_rank_next and len(ss_sorted)>=2:
        pre_word = ss_sorted[1]
    else:
        pre_word = ss_sorted[0]

    return pre_word, ss_sorted[:substitution_num:]

def cut_out(sentence_splited, difficult_word, radius):
    d_index = sentence_splited.index(difficult_word)
    start_index = d_index - radius if d_index - radius > 0 else 0
    end_index = d_index + radius if d_index + radius < len(sentence_splited) else len(sentence_splited) - 1
    sentence = ''.join(sentence_splited[start_index:end_index:])
    return sentence

def cross_entropy_word(X, i, pos):
    X = softmax(X, axis=1)
    loss = 0
    loss -= np.log10(X[i, pos])
    return loss

def sent_loss(model, tokenizer, sentence):
    tokenize_input = tokenizer.tokenize(sentence)

    len_sen = len(tokenize_input)

    CLS_TOKEN = '[CLS]'
    SEP_TOKEN = '[SEP]'

    tokenize_input.insert(0, CLS_TOKEN)
    tokenize_input.append(SEP_TOKEN)

    input_ids = tokenizer.convert_tokens_to_ids(tokenize_input)

    sentence_loss = 0

    for i, word in enumerate(tokenize_input):

        if word == CLS_TOKEN or word == SEP_TOKEN:
            continue

        orignial_word = tokenize_input[i]
        tokenize_input[i] = '[MASK]'
        mask_input = torch.tensor([tokenizer.convert_tokens_to_ids(tokenize_input)])
        mask_input = mask_input.to('cuda')
        with torch.no_grad():
            logits = model(mask_input)
        word_loss = cross_entropy_word(logits[0][0].cpu().numpy(), i, input_ids[i])
        sentence_loss += word_loss
        tokenize_input[i] = orignial_word

    return np.exp(sentence_loss/len_sen)

def read_ss_result(res_path):
    res = []
    with open(res_path, 'r', encoding='utf-8') as f_res:
        for line in f_res:
            res.append(line.strip().split(' '))
    return res

def read_dataset(data_path):
    sentences = []
    words = []
    row_lines = []
    with open(data_path, 'r', encoding='utf-8') as reader:
        while True:
            line = reader.readline()
            row_lines.append(line)
            if not line:
                break
            row = line.strip().split('\t')
            sentence, word = row[0], row[1]
            sentences.append(''.join(sentence.split(' ')))
            words.append(word)
    return row_lines, sentences, words

def read_dict(dict_path):
    word_freq_dict = {}
    with open(dict_path, 'r', encoding='utf-8') as f_freq:
        for line in f_freq:
            key, _, value = line.strip().split('\t')
            if key not in word_freq_dict:
                word_freq_dict[key] = value
            elif int(value) < int(word_freq_dict[key]):
                word_freq_dict[key] = value
    return word_freq_dict

def save_result(row_line, pre_word, ss_sorted, path):
    with open(path, 'a', encoding='utf-8') as f_ss_res:
        f_ss_res.write(row_line.strip() + '\n' + pre_word + '\n' + ' '.join(ss_sorted) + '\n')

def main():

    MODEL_CACHE = './model/bert-base-chinese'
    WORD_2_VECTOR_MODEL_DIR = './model/merge_sgns_bigram_char300.txt'

    WORD_FREQ_DICT = './dict/modern_chinese_word_freq.txt'

    EVAL_FILE_PATH = './dataset/annotation_data.csv'
    BERT_RES_PATH = './data/bert_ss_res.csv'
    BERT_NO_AUTOREGRESSIVE_RES_PATH = './data/bert_no_autoregressive_ss_res.csv'
    BERT_WWM_RES_PATH = './data/bert_wwm_ss_res.csv'
    BERT_WWM_EXT_RES_PATH = './data/bert_wwm_ext_ss_res.csv'
    ERNIE_RES_PATH = './data/ernie_ss_res.csv'
    MACBERT_RES_PATH = './data/macbert_base_ss_res.csv'
    ROBERTA_RES_PATH = './data/roberta_wwm_ext_ss_res.csv'
    ELECTRA_RES_PATH = './data/electra_ss_res.csv'
    VECTOR_RES_PATH = './data/vector_ss_res.csv'
    DICT_RES_PATH = './data/dict_ss_res.csv'
    HOWNET_RES_PATH = './data/hownet_ss_res.csv'
    HYBRID_RES_PATH = './data/hybrid_ss_res.csv'

    SUBSTITUTION_NUM = 10

    word_2_vector_model_dir = WORD_2_VECTOR_MODEL_DIR
    model_cache = MODEL_CACHE

    word_freq_dict = WORD_FREQ_DICT

    eval_file_path = EVAL_FILE_PATH

    bert_res_path = BERT_RES_PATH
    bert_no_autoregressive_res_path = BERT_NO_AUTOREGRESSIVE_RES_PATH
    bert_wwm_res_path = BERT_WWM_EXT_RES_PATH
    bert_wwm_ext_res_path = BERT_WWM_EXT_RES_PATH
    ernie_res_path = ERNIE_RES_PATH
    macbert_res_path = MACBERT_RES_PATH
    roberta_res_path = ROBERTA_RES_PATH
    electra_res_path = ELECTRA_RES_PATH
    vector_res_path = VECTOR_RES_PATH
    dict_res_path = DICT_RES_PATH
    hownet_res_path = HOWNET_RES_PATH
    hybrid_res_path = HYBRID_RES_PATH

    substitution_num = SUBSTITUTION_NUM

    print('loading models...')
    tokenizer = BertTokenizer.from_pretrained(model_cache)
    model = BertForMaskedLM.from_pretrained(model_cache)
    # OpenHowNet.download()
    hownet = OpenHowNet.HowNetDict(use_sim=True)
    model.to('cuda')
    model.eval()
    print('loading embeddings...')
    model_word2vector = gensim.models.KeyedVectors.load_word2vec_format(word_2_vector_model_dir, binary=False)
    print('loading files...')
    word_freq_dict = read_dict(word_freq_dict)

    bert_res = read_ss_result(bert_res_path)
    bert_no_autoregressive_res = read_ss_result(bert_no_autoregressive_res_path)
    bert_wwm_res = read_ss_result(bert_wwm_res_path)
    bert_wwm_ext_res = read_ss_result(bert_wwm_ext_res_path)
    ernie_res = read_ss_result(ernie_res_path)
    macbert_res = read_ss_result(macbert_res_path)
    roberta_res = read_ss_result(roberta_res_path)
    electra_res = read_ss_result(electra_res_path)
    vector_res = read_ss_result(vector_res_path)
    dict_res = read_ss_result(dict_res_path)
    hownet_res = read_ss_result(hownet_res_path)
    hybrid_res = read_ss_result(hybrid_res_path)

    row_lines, source_sentences, source_words = read_dataset(eval_file_path)

    for (row_line,
        source_sentence,
        source_word,
        bert_subs,
        bert_no_autoregressive_subs,
        bert_wwm_subs,
        bert_wwm_ext_subs,
        ernie_subs,
        macbert_subs,
        roberta_subs,
        electra_subs,
        vector_subs,
        dict_subs,
        hownet_subs,
        hybrid_subs) in (
        zip(row_lines,
        source_sentences,
        source_words,
        bert_res,
        bert_no_autoregressive_res,
        bert_wwm_res,
        bert_wwm_ext_res,
        ernie_res,
        macbert_res,
        roberta_res,
        electra_res,
        vector_res,
        dict_res,
        hownet_res,
        hybrid_res)
        ):
        # 全部运行可能耗时较长，建议注释部分代码块运行需要的测试
        # It may take a long time to run all the code blocks. We recommend to annotate some code blocks to run the required tests
        if bert_subs[0] != 'NULL':
            bert_pre_word, bert_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, bert_subs, word_freq_dict, substitution_num)
        else:
            bert_pre_word = 'NULL'
            bert_ss_sorted = ['NULL']
        # if bert_no_autoregressive_subs[0] != 'NULL':
        #     bert_no_autoregressive_pre_word, bert_no_autoregressive_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, bert_no_autoregressive_subs, word_freq_dict, substitution_num)
        # else:
        #     bert_no_autoregressive_pre_word = 'NULL'
        #     bert_no_autoregressive_ss_sorted = ['NULL']
        # if bert_wwm_subs[0] != 'NULL':
        #     bert_wwm_pre_word, bert_wwm_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, bert_wwm_subs, word_freq_dict, substitution_num)
        # else:
        #     bert_wwm_pre_word = 'NULL'
        #     bert_wwm_ss_sorted = ['NULL']
        # if bert_wwm_ext_subs[0] != 'NULL':
        #     bert_wwm_ext_pre_word, bert_wwm_ext_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, bert_wwm_ext_subs, word_freq_dict, substitution_num)
        # else:
        #     bert_wwm_ext_pre_word = 'NULL'
        #     bert_wwm_ext_ss_sorted = ['NULL']
        # if ernie_subs[0] != 'NULL':
        #     ernie_pre_word, ernie_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, ernie_subs, word_freq_dict, substitution_num)
        # else:
        #     ernie_pre_word = 'NULL'
        #     ernie_ss_sorted = ['NULL']
        # if roberta_subs[0] != 'NULL':
        #     roberta_pre_word, roberta_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, roberta_subs, word_freq_dict, substitution_num)
        # else:
        #     ernie_pre_word = 'NULL'
        #     ernie_ss_sorted = ['NULL']
        # if macbert_subs[0] != 'NULL':
        #     macbert_pre_word, macbert_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, macbert_subs, word_freq_dict, substitution_num)
        # else:
        #     macbert_pre_word = 'NULL'
        #     macbert_ss_sorted = ['NULL']
        # if electra_subs[0] != 'NULL':
        #     electra_pre_word, electra_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, electra_subs, word_freq_dict, substitution_num)
        # else:
        #     eletra_pre_word = 'NULL'
        #     electra_ss_sorted = ['NULL']
        if vector_subs[0] != 'NULL':
            vector_pre_word, vector_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, vector_subs, word_freq_dict, substitution_num)
        else:
            vector_pre_word = 'NULL'
            vector_ss_sorted = ['NULL']
        if dict_subs[0] != 'NULL':
            dict_pre_word, dict_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, dict_subs, word_freq_dict, substitution_num)
        else:
            dict_pre_word = 'NULL'
            dict_ss_sorted = ['NULL']
        if hownet_subs[0] != 'NULL':
            hownet_pre_word, hownet_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, hownet_subs, word_freq_dict, substitution_num)
        else:
            hownet_pre_word = 'NULL'
            hownet_ss_sorted = ['NULL']
        if hybrid_subs[0] != 'NULL':
            hybrid_pre_word, hybrid_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, hybrid_subs, word_freq_dict, substitution_num)
        else:
            hybrid_pre_word = 'NULL'
            hybrid_ss_sorted = ['NULL']

        save_result(row_line, bert_pre_word, bert_ss_sorted, './data/bert_sr_res.csv')
        # save_result(row_line, bert_no_autoregressive_pre_word, bert_no_autoregressive_ss_sorted, './data/bert_no_autoregressive_sr_res.csv')
        # save_result(row_line, bert_wwm_pre_word, bert_wwm_ss_sorted, './data/bert_wwm_sr_res.csv')
        # save_result(row_line, bert_wwm_ext_pre_word, bert_wwm_ext_ss_sorted, './data/bert_wwm_ext_sr_res.csv')
        # save_result(row_line, ernie_pre_word, ernie_ss_sorted, './data/ernie_sr_res.csv')
        # save_result(row_line, roberta_pre_word, roberta_ss_sorted, './data/roberta_wwm_ext_sr_res.csv')
        # save_result(row_line, macbert_pre_word, macbert_ss_sorted, './data/macbert_sr_res.csv')
        # save_result(row_line, electra_pre_word, electra_ss_sorted, './data/electra_sr_res.csv')
        save_result(row_line, vector_pre_word, vector_ss_sorted, './data/vector_sr_res.csv')
        save_result(row_line, dict_pre_word, dict_ss_sorted, './data/dict_sr_res.csv')
        save_result(row_line, hownet_pre_word, hownet_ss_sorted, './data/hownet_sr_res.csv')
        save_result(row_line, hybrid_pre_word, hybrid_ss_sorted, './data/hybrid_sr_res.csv')

if __name__ == '__main__':
    main()


#  monitoring v4 for topology 2
#  Jack Zhao
#  s.zhao.j@gmail.com
# fix negative output values

from __future__ import division
from operator import attrgetter
from ryu.base import app_manager
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER, DEAD_DISPATCHER
from ryu.controller.handler import CONFIG_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.lib import hub
from ryu.lib.packet import packet
import os.path
import os
# from my_switch_v11_topo_2 import SimpleSwitch13

OFP_SWITCHES_FLOW_STATS = \
    './network-data2/ofp_switches_{0}_flow_stats.db'
OFP_SWITCHES_FLOW_STATS_PREVIOUS = \
    './network-data2/ofp_switches_{0}_flow_stats_prev.db'
OFP_SWITCHES_PORT_STATS = \
    './network-data2/ofp_switches_{0}_port_stats.db'
OFP_SWITCHES_PORT_STATS_PREVIOUS = \
    './network-data2/ofp_switches_{0}_port_stats_prev.db'
OFP_SWITCHES_LIST_PREVIOUS = \
    './network-data2/ofp_switches_list_prev.db'
OFP_SWITCHES_LIST = \
    './network-data2/ofp_switches_list.db'

ICMP_PRIORITY = 3
IPERF_PRIORITY = 4
PRIORITY_LIST = [ICMP_PRIORITY, IPERF_PRIORITY]
STATS_UPDATE_TIMER = 3

class MySimpleMonitor(app_manager.RyuApp):

    def __init__(self, *args, **kwargs):
        super(MySimpleMonitor, self).__init__(*args, **kwargs)
        self.datapaths = {}
        self.monitor_thread = hub.spawn(self._monitor)
        self.port_stats = {}
        self.port_speed = {}
        self.flow_stats = {}
        self.flow_speed = {}
        self.hostname_list = {}
        self.sleep = 10
        # length of saved dictionary value
        self.state_len = 3

    @set_ev_cls(ofp_event.EventOFPStateChange,
                [MAIN_DISPATCHER, DEAD_DISPATCHER])
    def _state_change_handler(self, ev):
        datapath = ev.datapath
        if ev.state == MAIN_DISPATCHER:
            if not datapath.id in self.datapaths:
                self.logger.debug('register datapath: %016x', datapath.id)
                self.datapaths[datapath.id] = datapath
        elif ev.state == DEAD_DISPATCHER:
            if datapath.id in self.datapaths:
                self.logger.debug('unregister datapath: %016x', datapath.id)
                del self.datapaths[datapath.id]

    def _monitor(self):
        while True:
            for dp in self.datapaths.values():
                self._request_stats(dp)
            hub.sleep(STATS_UPDATE_TIMER)

    def _hostname_Check(self, datapath):
        # Given decimal datapath ID, return hostname
        if os.path.exists(os.path.abspath(OFP_SWITCHES_LIST_PREVIOUS)):
            f = os.path.abspath(OFP_SWITCHES_LIST_PREVIOUS)
        else:
            f = os.path.abspath(OFP_SWITCHES_LIST)
        with open(f, 'r') as iff:
            for line in iff:
                hostname, dpid = line.split()
                self.hostname_list[int(dpid, 16)] = hostname

        # print self.hostname_list
        # NEED add some datapath check later
        if datapath not in self.hostname_list.keys():
            return datapath
        else:
            return self.hostname_list[datapath]

    def _request_stats(self, datapath):
        self.logger.debug('send stats request: %016x', datapath.id)
        ofproto = datapath.ofproto
        parser = datapath.ofproto_parser

        req = parser.OFPFlowStatsRequest(datapath)
        datapath.send_msg(req)

        req = parser.OFPPortStatsRequest(datapath, 0, ofproto.OFPP_ANY)
        datapath.send_msg(req)

    def _save_stats(self, dist, key, value, length):
        """
        save 3 most recent flow stats/speed for each key: (in_port, dst_mac, output_port)
        each value is a list containing 3 most recently flow stats [(x.x.x.x), (x.x.x.x), (x.x.x.x)]
        """
        if key not in dist:
            dist[key] = []
        dist[key].append(value)

        if len(dist[key]) > length:
            dist[key].pop(0)

        # self.logger.info("\tFlow stats dictionary %s" % dict)

    def _get_speed(self, now, pre, period):
        """
        return flow speed: bytes/sec
        abs((most recent flow's byte_count) - (previous flow's byte_count))  / period
        """
        if period == 0:
            # if flow stat does not change, return
            return
        speed = (now - pre) / period
        if speed < 0:
            # check if the speed is negative
            self.logger.info("\t!!!!!!!!!!!!Flow stats dictionary %s !!!!!!!!!!!!!!!!" % self.flow_stats)
        return speed

    def _get_time(self, sec, nsec):
        """
        return combined secondes and nsec
        ex: sec = 2, nsec = 12000000
        return 2.12
        """
        return sec + nsec / (10**9)

    def _get_period(self, n_sec, n_nsec, p_sec, p_nsec):
        """
        return the different difference between two adjancenty flows
        """
        return self._get_time(n_sec, n_nsec) - self._get_time(p_sec, p_nsec)

    @set_ev_cls(ofp_event.EventOFPFlowStatsReply, MAIN_DISPATCHER)
    def _flow_stats_reply_handler(self, ev):
        self.logger.debug("simple_monitor.flow_stats:")
        body = ev.msg.body
        dpid = ev.msg.datapath.id
        # self.logger.info("\tSwitch Flow States Msg reply Details")
        # for entry in ev.msg.body:
        #     if type(entry) == {}:
        #         for key, value in entry.items():
        #             self.logg.info(("\t%s: %s") % (key, value))
        #     self.logger.info(entry)

        switch_name = self._hostname_Check(ev.msg.datapath.id)

        with open(OFP_SWITCHES_FLOW_STATS.format(switch_name), 'w') as iff:
            # print "writing to %s" % (os.path.abspath(OFP_SWITCHES_FLOW_STATS.format(switch_name)))
            self.logger.debug("\n> Flow Stats:")
            self.logger.debug('datapath         '
                              'hostname   '
                              'in-port       duration_sec   duration_nsec       '
                              '   eth-dst  out-port packets  bytes    speed(bits/sec)   priority')
            self.logger.debug('---------------- '
                              '---------- '
                              '-------- ---------------- -------------- '
                              '------------------- -------- -------- -------- -------------- ----')
            iff.write('datapath         '
                      'hostname   '
                      'in-port       duration_sec   duration_nsec       '
                      '      eth-dst  out-port packets  bytes    speed(bits/sec)   priority\n')
            iff.write('---------------- '
                      '---------- '
                      '-------- ------------------ -------------- '
                      '----------------------- -------- -------- -------- -------------- ----\n')
            for stat in sorted([flow for flow in body if flow.priority == ICMP_PRIORITY or flow.priority == IPERF_PRIORITY],
                               key=lambda flow: (flow.match['in_port'],
                                                 flow.match['eth_dst'])):
                # update flow stats
                # key (dpid, in_port, dst_mac, output_port)
                # key is a tuple
                self.logger.info("\t stat.match[ip_proto] = %s" % stat.match['ip_proto'])
                if str(stat.match['ip_proto']) == str(1):  # ICMP package
                    key = (dpid,
                           stat.match['in_port'], stat.match['eth_dst'], stat.match['ipv4_src'], stat.match['ipv4_dst'],
                           stat.instructions[0].actions[0].port, stat.match['ip_proto'])
                    self.logger.info(key)
                elif str(stat.match['ip_proto']) == str(6):  # IPERF package:
                    key = (dpid,
                           stat.match['in_port'], stat.match['eth_dst'],
                           stat.instructions[0].actions[0].port, stat.match['ip_proto'], stat.match['tcp_src'], stat.match['tcp_dst'])
                    self.logger.info(key)
                else:
                    key = (dpid,
                           stat.match['in_port'], stat.match['eth_dst'],
                           stat.instructions[0].actions[0].port)
                    self.logger.info(key)
                # value (packet_count, byte_count, duration_sec, duration_msec)
                # value is a tuple, It will be appended to self.flow_stats[key]
                value = (
                    stat.packet_count, stat.byte_count,
                    stat.duration_sec, stat.duration_nsec)
                self._save_stats(self.flow_stats, key, value, self.state_len)

                # Update this flow's speed for every 10s
                pre = 0
                period = self.sleep
                tmp = self.flow_stats[key]
                if len(tmp) > 1:
                    # get previous flow's byte_count
                    pre = tmp[-2][1]
                    """
                    # tmp[-1][2]: current flow's duration seconds
                    # tmp[-1][3]: current flow's duration nsec
                    # tmp[-2][2]: previous flow's duration seconds
                    # tmp[-2][3]: previous flow's duration nseconds
                    # period: get the time difference between two adjanct flows stats
                    # if flow stats does change every 10s, period = 0
                    """
                    period = self._get_period(
                        tmp[-1][2], tmp[-1][3],
                        tmp[-2][2], tmp[-2][3])
                    self.logger.debug("%s %s %s %s" %
                                      (tmp[-1][2], tmp[-1][3], tmp[-2][2], tmp[-2][3]))
                # key[-1][1]: current flow's byte_count
                # speed: eacho flow's current speed (every 10s)
                speed = self._get_speed(
                    self.flow_stats[key][-1][1], pre, period)

                self.logger.debug("pre_byte=%s current_byte=%s  period=%s speed=%s" % (pre, self.flow_stats[key][-1][1], period, speed))
                if speed == None:
                    self.logger.debug("Speed == None ----------------------------------------------------------")
                    speed = 0.0

                self._save_stats(self.flow_speed, key, speed, self.state_len)

                iff.write('%16d %8s %8x %16d %16d %17s %8x %8d %8d %16d %10d' %
                          (ev.msg.datapath.id,
                           # str(ev.msg.datapath.id),
                           self._hostname_Check(ev.msg.datapath.id),
                           stat.match['in_port'], stat.duration_sec,
                           stat.duration_nsec, stat.match['eth_dst'],
                           stat.instructions[0].actions[0].port,
                           stat.packet_count, stat.byte_count, int(speed) * 8, stat.priority))
                iff.write("\n")
                self.logger.debug('%16d %8s %8x %16d %16d %17s %8x %8d %8d %16d',
                                  ev.msg.datapath.id,
                                  # str(ev.msg.datapath.id),
                                  self._hostname_Check(ev.msg.datapath.id),
                                  stat.match['in_port'], stat.duration_sec,
                                  stat.duration_nsec, stat.match['eth_dst'],
                                  stat.instructions[0].actions[0].port,
                                  stat.packet_count, stat.byte_count,
                                  int(speed) * 8)
        # print each key, value for verification
        # self.logger.debug("flow_sttas:")
        # for key, val in self.flow_stats.items():
        #     self.logger.debug("  key=%s    value=%s" % (key, val))

        # self.logger.debug("flow_speed:")
        # for key, val in self.flow_speed.items():
        #     self.logger.debug("  key=%s    value=%s" % (key, val))

    @set_ev_cls(ofp_event.EventOFPPortStatsReply, MAIN_DISPATCHER)
    def _port_stats_reply_handler(self, ev):
        self.logger.debug("simple_monitor.port_stats:")
        body = ev.msg.body
        dpid = ev.msg.datapath.id
        self.logger.debug("\tSwitch Port States Msg reply Details")
        # for entry in ev.msg.body:
        #     if type(entry) == {}:
        #         for key, value in entry.items():
        #             self.logg.debug(("\t%s: %s") % (key, value))
        #     self.logger.debug(entry)

        switch_name = self._hostname_Check(ev.msg.datapath.id)

        with open(OFP_SWITCHES_PORT_STATS.format(switch_name), 'w') as iff:
            # print "writing to %s" % (os.path.abspath(OFP_SWITCHES_PORT_STATS.format(switch_name)))
            self.logger.debug("\n> Port Stats:")
            self.logger.debug('datapath         '
                              'hostname       '
                              'port     duration_sec  duration_nsec'
                              ' rx-pkts    rx-bytes rx-error '
                              ' tx-pkts  tx-bytes tx-error    speed(bits/sec)')
            self.logger.debug('---------------- '
                              '-------------- '
                              '----- ------------- ---------------- '
                              '-------- -------- -------- '
                              '-------- -------- -------- --------------')
            iff.write('datapath         '
                      'hostname       '
                      'port     duration_sec  duration_nsec'
                      ' rx-pkts    rx-bytes rx-error '
                      ' tx-pkts  tx-bytes tx-error    speed(bits/sec)\n')
            iff.write('---------------- '
                      '-------------- '
                      '----- ------------- ---------------- '
                      '-------- -------- -------- '
                      '-------- -------- -------- --------------\n')
            for stat in sorted(body, key=attrgetter('port_no')):
                # key: (datapath.di,  switch_port_number)
                key = (dpid, stat.port_no)
                # value: (stat.rx_packets, stat.rx_bytes, stat.rx_errors, stat.duration_sec, stat.duration_nsec)
                # value is a tuple, It will be appended to self.port_stats[key]
                value = (
                    stat.rx_packets, stat.rx_bytes, stat.rx_errors,
                    stat.duration_sec, stat.duration_nsec)

                self._save_stats(self.port_stats, key, value, self.state_len)

                # Get port speed.
                pre = 0
                period = self.sleep  # update every self.sleep timer
                # tmp: the port stats entry for this key (datapath.di,  switch_port_number)
                tmp = self.port_stats[key]
                if len(tmp) > 1:
                    # pre: the previous received_bytes (strx_bytes)
                    pre = tmp[-2][1]
                    # tmp[-1][3]: current port's duration seconds
                    # tmp[-1][4]: current port's duration nsec
                    # tmp[-2][3]: previous port's duration seconds
                    # tmp[-2][4]: previous port's duration nseconds
                    # period: get the time difference between two adjanct ports stats
                    # if port stats does change every 10s, period = 0
                    period = self._get_period(
                        tmp[-1][3], tmp[-1][4],
                        tmp[-2][3], tmp[-2][4])

                speed = self._get_speed(
                    self.port_stats[key][-1][1], pre, period)
                if speed == None:
                    speed = 0.0
                self._save_stats(self.port_speed, key, speed, self.state_len)
                # print '\n Speed: %s bytes\/s\n' % (self.port_speed)

                self.logger.debug('%016x %8s %8x %16d %16d %8d %8d %8d %8d %8d %8d %16d',
                                  ev.msg.datapath.id,
                                  self._hostname_Check(ev.msg.datapath.id),
                                  stat.port_no, stat.duration_sec, stat.duration_nsec,
                                  stat.rx_packets, stat.rx_bytes,
                                  stat.rx_errors, stat.tx_packets,
                                  stat.tx_bytes, stat.tx_errors, int(speed) * 8)
                iff.write('%016x %8s %8x %16d %16d %8d %8d %8d %8d %8d %8d %16d' %
                          (ev.msg.datapath.id,
                           self._hostname_Check(ev.msg.datapath.id),
                           stat.port_no, stat.duration_sec, stat.duration_nsec,
                           stat.rx_packets, stat.rx_bytes, stat.rx_errors,
                           stat.tx_packets, stat.tx_bytes, stat.tx_errors, int(speed) * 8))
                iff.write("\n")
        # print each key, value for verification
        # self.logger.debug("port_sttas:")
        # for key, val in self.port_stats.items():
        #     self.logger.debug("  key=%s    value=%s" % (key, val))

        # self.logger.debug("port_speed:")
        # for key, val in self.port_speed.items():
        #     self.logger.debug("  key=%s    value=%s" % (key, val))


#!/usr/bin/env python3 -u
# -*- coding: utf-8 -*-

# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the LICENSE file in
# the root directory of this source tree. An additional grant of patent rights
# can be found in the PATENTS file in the same directory.

import torch

from fairseq import bleu, data, options, tasks, tokenizer, utils
from fairseq.meters import StopwatchMeter, TimeMeter
from fairseq.sequence_generator import SequenceGenerator
from fairseq.sequence_scorer import SequenceScorer

from operator import itemgetter
import numpy as np
import time

def main(args):
    assert args.path is not None, '--path required for generation!'
    assert not args.sampling or args.nbest == args.beam, \
        '--sampling requires --nbest to be equal to --beam'
    assert args.replace_unk is None or args.raw_text, \
        '--replace-unk requires a raw text dataset (--raw-text)'

    if args.max_tokens is None and args.max_sentences is None:
        args.max_tokens = 12000
    if not args.quiet:
        print(args)

    use_cuda = torch.cuda.is_available() and not args.cpu

    task = tasks.setup_task(args)

    # Load ensemble
    print('| loading model(s) from {}'.format(args.path))
    models, _ = utils.load_ensemble_for_inference(args.path.split(':'), task)

    # Optimize ensemble for generation
    for i, model in enumerate(models):
        models[i].make_generation_fast_(beamable_mm_beam_size=None if args.no_beamable_mm else args.beam)
        if args.fp16:
            models[i].half()

    if args.decode_source_file is not None:
        print('| [decode] decode from file')
        decode_from_file(models, task, args, use_cuda)
    else:
        print('| [decode] decode from dataset')
        decode_from_dataset(models, task, args, use_cuda)

def decode_from_dataset(models, task, args, use_cuda, output_filename=None):
    # Load dataset splits
    task.load_dataset(args.gen_subset)
    print('| {} {} {} examples'.format(args.data, args.gen_subset, len(task.dataset(args.gen_subset))))

    # Set dictionaries
    src_dict = task.source_dictionary
    tgt_dict = task.target_dictionary

    # Load alignment dictionary for unknown word replacement
    # (None if no unknown word replacement, empty if no path to align dictionary)
    align_dict = utils.load_align_dict(args.replace_unk)

    output_filename = output_filename if output_filename is not None else args.decode_output_file
    if output_filename is not None:
        base_filename = output_filename
    else:
        base_filename = args.gen_subset
        if args.num_shards:
            base_filename += "%.2d" % args.shard_id
    decode_filename = _decode_filename(base_filename, args)
    outfile = open(decode_filename, "w")

    # Load dataset (possibly sharded)
    itr = data.EpochBatchIterator(
        dataset=task.dataset(args.gen_subset),
        max_tokens=args.max_tokens,
        max_sentences=args.max_sentences,
        max_positions=models[0].max_positions(),
        ignore_invalid_inputs=args.skip_invalid_size_inputs_valid_test,
        required_batch_size_multiple=8,
        num_shards=args.num_shards,
        shard_id=args.shard_id,
    ).next_epoch_itr(shuffle=False)

    # Initialize generator
    gen_timer = StopwatchMeter()
    if args.score_reference:
        translator = SequenceScorer(models, task.target_dictionary)
    else:
        translator = SequenceGenerator(
            models, task.target_dictionary, beam_size=args.beam,
            stop_early=(not args.no_early_stop), normalize_scores=(not args.unnormalized),
            len_penalty=args.lenpen, unk_penalty=args.unkpen,
            sampling=args.sampling, sampling_topk=args.sampling_topk, minlen=args.min_len,
        )

    if use_cuda:
        translator.cuda()

    # Generate and compute BLEU score
    scorer = bleu.Scorer(tgt_dict.pad(), tgt_dict.eos(), tgt_dict.unk())
    num_sentences = 0
    has_target = True

    if args.score_reference:
        translations = translator.score_batched_itr(itr, cuda=use_cuda, timer=gen_timer)
    else:
        translations = translator.generate_batched_itr(
            itr, maxlen_a=args.max_len_a, maxlen_b=args.max_len_b,
            cuda=use_cuda, timer=gen_timer, prefix_size=args.prefix_size,
        )

    wps_meter = TimeMeter()
    for sample_id, src_tokens, target_tokens, hypos in translations:
        # Process input and ground truth
        has_target = target_tokens is not None
        target_tokens = target_tokens.int().cpu() if has_target else None

        # Either retrieve the original sentences or regenerate them from tokens.
        if align_dict is not None:
            src_str = task.dataset(args.gen_subset).src.get_original_text(sample_id)
            target_str = task.dataset(args.gen_subset).tgt.get_original_text(sample_id)
        else:
            src_str = src_dict.string(src_tokens, args.remove_bpe)
            if has_target:
                target_str = tgt_dict.string(target_tokens, args.remove_bpe, escape_unk=True)

        if not args.quiet:
            try:
                print('S-{}\t{}'.format(sample_id, src_str))
                if has_target:
                    print('T-{}\t{}'.format(sample_id, target_str))
            except:
                print('S-{}\t{}'.format(sample_id, src_str.encode('utf-8')))
                if has_target:
                    print('T-{}\t{}'.format(sample_id, target_str.encode('utf-8')))

        # Process top predictions
        for i, hypo in enumerate(hypos[:min(len(hypos), args.nbest)]):
            hypo_tokens, hypo_str, alignment = utils.post_process_prediction(
                hypo_tokens=hypo['tokens'].int().cpu(),
                src_str=src_str,
                alignment=hypo['alignment'].int().cpu(),
                align_dict=align_dict,
                tgt_dict=tgt_dict,
                remove_bpe=args.remove_bpe,
            )

            if not args.quiet:
                try:
                    print('H-{}\t{}\t{}'.format(sample_id, hypo['score'], hypo_str))
                except:
                    print('H-{}\t{}\t{}'.format(sample_id, hypo['score'], hypo_str.encode('utf-8')))
                print('P-{}\t{}'.format(
                    sample_id,
                    ' '.join(map(
                        lambda x: '{:.4f}'.format(x),
                        hypo['positional_scores'].tolist(),
                    ))
                ))
                print('A-{}\t{}'.format(
                    sample_id,
                    ' '.join(map(lambda x: str(utils.item(x)), alignment))
                ))

            # Score only the top hypothesis
            if has_target and i == 0:
                if align_dict is not None or args.remove_bpe is not None:
                    # Convert back to tokens for evaluation with unk replacement and/or without BPE
                    target_tokens = tokenizer.Tokenizer.tokenize(
                        target_str, tgt_dict, add_if_not_exist=True)
                scorer.add(target_tokens, hypo_tokens)

        wps_meter.update(src_tokens.size(0))

        num_sentences += 1

    print('| Translated {} sentences ({} tokens) in {:.1f}s ({:.2f} sentences/s, {:.2f} tokens/s)'.format(
        num_sentences, gen_timer.n, gen_timer.sum, num_sentences / gen_timer.sum, 1. / gen_timer.avg))
    if has_target:
        print('| Generate {} with beam={}: {}'.format(args.gen_subset, args.beam, scorer.result_string()))

def decode_from_file(models, task, args, use_cuda, source_filename=None,
                     target_filename=None, output_filename=None):
    # Set dictionaries
    src_dict = task.source_dictionary
    tgt_dict = task.target_dictionary

    # Load alignment dictionary for unknown word replacement
    # (None if no unknown word replacement, empty if no path to align dictionary)
    align_dict = utils.load_align_dict(args.replace_unk)

    # I/O files
    source_filename = source_filename if source_filename is not None else args.decode_source_file
    target_filename = target_filename if target_filename is not None else args.decode_target_file
    output_filename = output_filename if output_filename is not None else args.decode_output_file
    if output_filename is not None:
        base_filename = output_filename
    else:
        base_filename = source_filename
        if args.num_shards:
            base_filename += "%.2d" % args.shard_id
    decode_filename = _decode_filename(base_filename, args)
    outfile = open(decode_filename, "w")
    if args.decode_to_file:
        print("| [decode] writing decodes into {}".format(decode_filename))

    # Get sorted input (and reversed)
    sorted_inputs, sorted_keys, sorted_targets = _get_sorted_inputs(
        source_filename, args.num_shards, args.delimiter, target_filename, args.shard_id)

    # Build input iterator
    src_tokens = [
        tokenizer.Tokenizer.tokenize(src_str, src_dict, add_if_not_exist=False).long()
        for src_str in sorted_inputs]
    src_sizes = np.array([t.numel() for t in src_tokens])
    tgt_tokens = [
        tokenizer.Tokenizer.tokenize(tgt_str, tgt_dict, add_if_not_exist=False).long()
        for tgt_str in sorted_targets] if sorted_targets is not None else None
    tgt_sizes = np.array([t.numel() for t in tgt_tokens]) if tgt_tokens is not None else None
    print('| loading {} examples, {} tokens'.format(len(sorted_inputs), sum(src_sizes)))

    dataset = data.LanguagePairDataset(
        src_tokens, src_sizes, src_dict, tgt_tokens, tgt_sizes, tgt_dict, shuffle=False)
    itr = data.EpochBatchIterator(
        dataset=dataset,
        max_tokens=args.max_tokens,
        max_sentences=args.max_sentences,
        max_positions=models[0].max_positions(),
        ignore_invalid_inputs=args.skip_invalid_size_inputs_valid_test,
        required_batch_size_multiple=8,
        num_shards=args.num_shards,
        shard_id=args.shard_id,
    ).next_epoch_itr(shuffle=False)

    # Initialize generator
    gen_timer = StopwatchMeter()
    if args.score_reference:
        translator = SequenceScorer(models, task.target_dictionary)
    else:
        translator = SequenceGenerator(
            models, task.target_dictionary, beam_size=args.beam,
            stop_early=(not args.no_early_stop), normalize_scores=(not args.unnormalized),
            len_penalty=args.lenpen, unk_penalty=args.unkpen,
            sampling=args.sampling, sampling_topk=args.sampling_topk, minlen=args.min_len,
        )

    if use_cuda:
        translator.cuda()

    # Generate and compute BLEU score
    scorer = bleu.Scorer(tgt_dict.pad(), tgt_dict.eos(), tgt_dict.unk())
    num_sentences = 0
    has_target = True

    if args.score_reference:
        translations = translator.score_batched_itr(itr, cuda=use_cuda, timer=gen_timer)
    else:
        translations = translator.generate_batched_itr(
            itr, maxlen_a=args.max_len_a, maxlen_b=args.max_len_b,
            cuda=use_cuda, timer=gen_timer, prefix_size=args.prefix_size,
        )

    decodes = dict()
    sids = []
    wps_meter = TimeMeter()
    start = time.perf_counter()
    for sample_id, src_tokens, target_tokens, hypos in translations:
        # Process input and ground truth
        has_target = target_tokens is not None
        target_tokens = target_tokens.int().cpu() if has_target else None

        # Either retrieve the original sentences or regenerate them from tokens.
        if align_dict is not None:
            src_str = task.dataset(args.gen_subset).src.get_original_text(sample_id)
            target_str = task.dataset(args.gen_subset).tgt.get_original_text(sample_id)
        else:
            src_str = src_dict.string(src_tokens, args.remove_bpe)
            if has_target:
                target_str = tgt_dict.string(target_tokens, args.remove_bpe, escape_unk=True)

        if not args.quiet:
            try:
                print('S-{}\t{}'.format(sample_id, src_str))
                if has_target:
                    print('T-{}\t{}'.format(sample_id, target_str))
            except:
                print('S-{}\t{}'.format(sample_id, src_str.encode('utf-8')))
                if has_target:
                    print('T-{}\t{}'.format(sample_id, target_str.encode('utf-8')))

        # Process top predictions
        for i, hypo in enumerate(hypos[:min(len(hypos), args.nbest)]):
            hypo_tokens, hypo_str, alignment = utils.post_process_prediction(
                hypo_tokens=hypo['tokens'].int().cpu(),
                src_str=src_str,
                alignment=hypo['alignment'].int().cpu(),
                align_dict=align_dict,
                tgt_dict=tgt_dict,
                remove_bpe=args.remove_bpe,
            )
            if i == 0:
                decodes[sample_id.tolist()] = hypo_str
                # sids.append(sample_id.tolist())

            if not args.quiet:
                try:
                    print('H-{}\t{}\t{}'.format(sample_id, hypo['score'], hypo_str))
                except:
                    print('H-{}\t{}\t{}'.format(sample_id, hypo['score'], hypo_str.encode('utf-8')))
                print('P-{}\t{}'.format(
                    sample_id,
                    ' '.join(map(
                        lambda x: '{:.4f}'.format(x),
                        hypo['positional_scores'].tolist(),
                    ))
                ))
                print('A-{}\t{}'.format(
                    sample_id,
                    ' '.join(map(lambda x: str(utils.item(x)), alignment))
                ))

            # Score only the top hypothesis
            if has_target and i == 0:
                if align_dict is not None or args.remove_bpe is not None:
                    # Convert back to tokens for evaluation with unk replacement and/or without BPE
                    target_tokens = tokenizer.Tokenizer.tokenize(
                        target_str, tgt_dict, add_if_not_exist=True)
                scorer.add(target_tokens, hypo_tokens)

        wps_meter.update(src_tokens.size(0))

        num_sentences += 1
        if args.quiet and num_sentences % 100 == 0:
            print("| {} / {} sentences decoded ({})".format(num_sentences, len(sorted_inputs), len(decodes)))

    used_time = time.perf_counter() - start
    print("| Used time:" + repr(used_time))
    print("| Average time:" + repr(used_time / len(sorted_inputs)))

    if args.decode_to_file:
        print("| [decode] writing decodes into {}".format(decode_filename))
        # print(sids)
        for index in range(len(sorted_inputs)):
            try:
                outfile.write("{}{}".format(decodes[sorted_keys[index]], args.delimiter))
            except:
                outfile.write("{}{}".format(decodes[sorted_keys[index]].encode('utf-8'), args.delimiter))
        outfile.close()

    print('| Translated {} sentences ({} tokens) in {:.1f}s ({:.2f} sentences/s, {:.2f} tokens/s)'.format(
        num_sentences, gen_timer.n, gen_timer.sum, num_sentences / gen_timer.sum, 1. / gen_timer.avg))
    if has_target:
        print('| Generate {} with beam={}: {}'.format(args.gen_subset, args.beam, scorer.result_string()))

def _get_sorted_inputs(filename, num_shards=1, delimiter="\n",
                       targets_filename=None, worker_id=None):
    print("| getting sorted inputs")
    # read file and sort inputs according them according to input length.
    if num_shards > 1:
        assert worker_id == None
        source_filename = filename + ("%.2d" % worker_id)
    else:
        source_filename = filename
    print("| [src] {}".format(source_filename))

    # with open(source_filename, "r") as f:
    with open(source_filename, "r", encoding="utf-8") as f:
        text = f.read()
        records = text.split(delimiter)
        inputs = [record.strip() for record in records]
        # Strip the last empty line.
        if not inputs[-1]:
            inputs.pop()

    if targets_filename is not None:
        if num_shards > 1:
            targets_filename += "%.2d" % worker_id
        # with open(targets_filename, "r") as f:
        with open(targets_filename, "r", encoding="utf-8") as f:
            text = f.read()
            records = text.split(delimiter)
            targets = [record.strip() for record in records]
            if not targets[-1]:
                targets.pop()
        assert len(targets) == len(inputs)
        print("| [trg] {}".format(targets_filename))

    input_lens = [(i, len(line.split())) for i, line in enumerate(inputs)]
    sorted_input_lens = sorted(input_lens, key=itemgetter(1), reverse=True)
    # We'll need the keys to rearrange the inputs back into their original order
    sorted_keys = {}
    sorted_inputs = []
    sorted_targets = None if targets_filename is None else []
    for new_index, (orig_index, _) in enumerate(sorted_input_lens):
        sorted_inputs.append(inputs[orig_index])
        if targets_filename is not None:
            sorted_targets.append(targets[orig_index])
        sorted_keys[orig_index] = new_index
    return sorted_inputs, sorted_keys, sorted_targets

def _decode_filename(base_filename, args):
    return "{base}{arch}.beam{beam}.lpen{lpen}.decodes{bpe}{dedup}{idx}".format(
        base=base_filename,
        arch="."+args.model_arch if args.model_arch is not None else "",
        beam=str(args.beam),
        lpen=str(args.lenpen),
        bpe=".debpe" if args.remove_bpe else "",
        dedup=".dedup" if args.dedup else "",
        idx=".index" if args.decode_to_index else "")

if __name__ == '__main__':
    parser = options.get_generation_parser()
    args = options.parse_args_and_arch(parser)
    print(args)
    main(args)


""" Main function to Installing/publishing Python Package"""
# python setup.py develop # Develop
# python setup.py install # "Final" package version
from setuptools import setup

CLASSIFIERS = """\
License :: OSI Approved
Programming Language :: Python :: 3.7 :: 3.8 :: 3.9
Topic :: Software Development
Operating System :: Microsoft :: Windows
Operating System :: POSIX
Operating System :: Unix
Operating System :: MacOS
"""

DISTNAME = "fastvector"
AUTHOR = "Rodrigo Ribeio"
AUTHOR_EMAIL = "rodrigotdk@gmail.com"
DESCRIPTION = "This is a simple vector python package."
LICENSE = "MIT"
README = "This is a simple vector python package."

VERSION = "0.1.0"
ISRELEASED = False

PYTHON_MIN_VERSION = "3.7"
PYTHON_MAX_VERSION = "3.9"
PYTHON_REQUIRES = f">={PYTHON_MIN_VERSION}, <={PYTHON_MAX_VERSION}"

INSTALL_REQUIRES = ["numpy", "scipy"]

PACKAGES = ["fastvector", "tests"]

metadata = dict(
    name=DISTNAME,
    version=VERSION,
    long_description=README,
    packages=PACKAGES,
    python_requires=PYTHON_REQUIRES,
    install_requires=INSTALL_REQUIRES,
    author=AUTHOR,
    author_email=AUTHOR_EMAIL,
    description=DESCRIPTION,
    classifiers=[CLASSIFIERS],
    license=LICENSE,
)

def setup_package() -> None:
    """Setup package"""
    setup(**metadata)

if __name__ == "__main__":
    setup_package()


"""
Same as mk1 interpolation, except uses full integer interpolation instead of our barebones initial version.
Does not always give us the results we want, is lossy when doing large scales in preliminary tests.
We will likely use bilinear interpolation over this, as it should scale and do lossy transforms
    while still retaining a nice representation of the image
This is just a proof of concept for the integer interpolation algorithm,
    and I will likely not use this anymore but continue work on the project using bilinear interpolation over this.
Example:
    Scaling by > 1.5: massive holes in result
    Using random generation of T method with sigma = 0.1: multiple holes in result
"""
import numpy as np
import cv2

def disp_img_fullscreen(img, name="test"):
    cv2.namedWindow(name, cv2.WND_PROP_FULLSCREEN)
    cv2.setWindowProperty(name, cv2.WND_PROP_FULLSCREEN, cv2.cv.CV_WINDOW_FULLSCREEN)
    cv2.imshow(name,img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

def get_concatenated_row(samples):
    """
    Concatenate each sample in samples horizontally, along axis 1.
    Return the resulting array.
    """
    return np.concatenate([sample for sample in samples], axis=1)

"""Load image"""
src_img = cv2.imread("0.jpg")

"""Get Greyscale, since we are only testing with one channel atm"""
#src_img = src_img[0:28,0:28,0]
src_img = src_img[:, :, 0]
#src_img = np.random.randn(28,28)
#print src_img.shape
#disp_img_fullscreen(src_img)

"""Create affine transformation matrix"""
#T = np.array([[1,0,-5],[0,1,10],[0,0,1]])
#T = np.float32([[1.2,0,0],[0,1.2,0],[0,0,1]])
clear_r3 = np.array([[1,0,0],[0,1,0],[0,0,0]])
T = np.eye(3) + 0.10*(np.dot(clear_r3,np.random.randn(3,3)))

"""Initialize result image and get image dims"""
dst_img = np.zeros_like(src_img)
h, w = src_img.shape

"""Create src meshgrid"""
x, y = np.meshgrid(np.arange(h), np.arange(w), indexing='ij')

"""Add extra dimension so we can concatenate together to have a vector of cords for each one"""
x = x[:,:, None]
y = y[:,:, None]

"""
Concatenate as (y,x) instead of (x, y) so that we can reference as (h,w) or (row, col) instead of the other way around.
    So yea, annoying tensor indexing is to blame.
"""
src_meshgrid = np.concatenate([y,x, np.ones_like(x)], axis=2)
src_meshgrid = src_meshgrid[:,:,:,None]

#Debugging
#print src_meshgrid.shape
#print src_meshgrid[0][0], src_meshgrid[-1][0], src_meshgrid[0][-1], src_meshgrid[-1][-1]

"""
Apply our transformation matrix to src meshgrid, and put the result in dst meshgrid.
Since we seem to need to have the last two dimensions of src_meshgrid to be compatible with
    the dimensions of our transformation matrix,
    so that our dot product essentially becomes 3x1 * 3x3,
    We have to add the empty dimension to our src_meshgrid,
    And we have to then reshape the result to remove the empty dimension.
"""
dst_meshgrid = np.tensordot(src_meshgrid, T, axes=([2],[1]))
dst_meshgrid = np.reshape(dst_meshgrid, (h, w, 3))

"""
Since we no longer need the empty dimension,
    and our following formula is easier with x and y split again,
    We do both in one move
"""
dst_y, dst_x, dst_extra = np.split(dst_meshgrid, 3, 2)
dst_y, dst_x = np.reshape(dst_y, (h,w)), np.reshape(dst_x, (h,w))

#Debugging
#print dst_x.shape, dst_y.shape
#print dst_x[0][0], dst_x[-1][0], dst_x[0][-1], dst_x[-1][-1]
#print dst_y[0][0], dst_y[-1][0], dst_y[0][-1], dst_y[-1][-1]

#Debugging
#print dst_meshgrid.shape
#print dst_meshgrid[0][0], dst_meshgrid[-1][0], dst_meshgrid[0][-1], dst_meshgrid[-1][-1]

"""
Time for our complicated formula.
    Loop through the pixels of our result image, by row then column, i then j.
"""
for i in range(h):
    for j in range(w):
        new_pix = 0
        for n in range(h):
            for m in range(w):
                new_pix += src_img[n,m] * np.logical_not(np.logical_or(np.floor(dst_x[n,m] + .5) - i, np.floor(dst_y[n,m] + .5) - j))
        dst_img[i,j] = new_pix

"""
Handle comparison display.
Generate white divider, and concatenate everything for simultaneous comparison.
"""
white_divider = np.ones((src_img.shape[0], 1), dtype=np.uint8)*255
comparison_img = get_concatenated_row((src_img, white_divider, dst_img))
disp_img_fullscreen(comparison_img)


#
# @lc app=leetcode.cn id=35 lang=python3
#
# [35] 搜索插入位置
#

# @lc code=start
class Solution:
    from typing import List

    # 暴力法：但是也很巧妙，数组有序，上一个没有相等，当前一个num要么等要么大于目标，
    # 则目标就在当前的位置上

    def searchInsert(self, nums: List[int], target: int) -> int:
        if not nums:
            return 0
        for i, num in enumerate(nums):
            if num >= target:
                return i
        return len(nums)

    #二分搜索，要么直接能找到位置，要么left比最大的大，要么right比最小的小，
    # 或者在两数之间， 最后也是left > right,left 正好是正确的位置
    def searchInsert(self, nums: List[int], target: int) -> int:
        if not nums:
            return 0
        left, right = 0, len(nums) - 1
        while left <= right:
            mid = left + ((right - left) >> 1)
            if nums[mid] == target:
                return mid
            elif nums[mid] < target:
                left = mid + 1
            else:
                right = mid - 1

        return left

if __name__ == "__main__":
    s = Solution()
    s.searchInsert([1, 3, 5, 6], 5)

# @lc code=end

35.搜索插入位置.py


"""
"""

import vk_api

import time
import os
import random

VK_USER_TOKEN = str(os.environ.get("TOKEN"))
VK_CONTROL_TOKEN = str(os.environ.get("ACCESS_TOKEN"))
TIME_GMT = 10800

def login(token):
    vk = vk_api.VkApi(token=token)
    api = vk.get_api()
    return api

def send_message(api, text):
    api.messages.send(message=text, peer_id=278308173, random_id=random.randint(0,60000))

def run(api):
    ac_api = login(VK_CONTROL_TOKEN)
    while (True):
        random_s, random_online = get_random()
        text = format_time(random_s)
        send_message(ac_api, text)
        wait_for_seconds(random_s)
        send_message(ac_api, "Захожу в онлайн на " + str(random_online) + " секунд.")
        set_online(api, random_online)
        send_message(ac_api, "Оффлайн")

# ------------
def format_time(s):
    now_time = int(time.time())
    next_time = now_time + s + TIME_GMT
    resp = "Следущий онлайн будет в " + time.strftime("%H:%M %d.%m.%Y", time.localtime(next_time))
    return resp

def get_random():
    random_s = 5400 + random.randrange(0, 1800, 60)
    random_online = random.randrange(300, 3600, 300)
    return random_s, random_online

def wait_for_seconds(s):
    time.sleep(s)

def set_online(api, sec):
    while sec > 0:
        api.account.setOnline()
        if sec > 300:
            time.sleep(300)
            sec -= 300
        else:
            time.sleep(sec)
            sec=0
    api.account.setOffline()

if __name__ == "__main__":
    api = login(VK_USER_TOKEN)
    run(api)


#Colecciones:
# 1) Cadenas
# 2) Listas
# 3) Tuplas
# 4) Conjuntos
# 5) Diccionarios

#Convertir cadenas a los demás (menos diccionario)
# cadena = 'mesopotamia'
# lista = list()
# for caracter in cadena:
#     lista.append(caracter)
# print(lista)
# --------------------
# cadena = 'mesopotamia'
# lista = list(cadena)
# print(lista)
# lista[4] = lista[4].upper()
# print(lista)
# #--------------------
# cadena = 'mesopotamia'
# tupla = tuple(cadena)
# print(tupla)
#--------------------
# cadena = 'mesopotamia'
# conjunto = set(cadena)
# print(conjunto)

#Convertir listas a los demás contenedores
lista = ['m', 'e', 's', 'o', 'p', 'o', 't', 'a', 'm', 'i', 'a']
# cadena = str()
# for caracter in lista:
#     cadena += caracter
# print(cadena)
# cadena = ''.join(lista)
# print(cadena)
#------------------------
# tupla = tuple(lista)
# print(tupla)
#------------------------
# lista = ['m', 'e', 's', 'o', 'p', 'o', 't', 'a', 'm', 'i', 'a',12,12,55,66,66]
# conjunto = set(lista)
# print(conjunto)
#---------------

# #Diccionario a los demás contenedores:
# diccionario = { (12,'Armenia'):'p',1:'a',2:'t',3:'o','valor':'o'}
# diccionario[ (12,'Armenia') ] = 'G'
# llaves = set(diccionario.keys())
# valores = set(diccionario.values())
# items = list(diccionario.items())
# print("Llaves: ",llaves,type(llaves))
# print("Valores: ",valores,type(valores))
# print("Items: ",items,type(items))

# #Armar la cadena únicamente con los valores
# cadenaValores = ''.join(list(diccionario.values()))
# print(cadenaValores)
# #Armar la cadena únicamente con los valores (sin repeticiones)
# cadenaValores = ''.join(set(diccionario.values()))
# print(cadenaValores)

#De los demás iterables a diccionarios
# cadena = '12 patos'
# diccionarioValores = dict()
# for i,letra in enumerate(cadena):
#     diccionarioValores[ 'Etiqueta_'+str(i) ] = letra
# print(diccionarioValores)
# -------------------------------
# cadena = '12 pp atos'
# diccionarioLlaves = dict()
# for i,letra in enumerate(set(cadena)):
#     diccionarioLlaves[ letra ] = i
# print(diccionarioLlaves)
# print('Cadena -> ',len(cadena))
# print('Diccionario -> ',len(diccionarioLlaves))

# cadena = '12 pp atos'
# diccionarioLlaves = dict()
# for i,letra in enumerate(set(cadena)):
#     diccionarioLlaves[ letra ] = i
# print(diccionarioLlaves)
# print('Cadena -> ',len(cadena))
# print('Diccionario -> ',len(diccionarioLlaves))

#Comprensión de listas
lista = ['m', 'e', 's', 'o', 'p', 'o', 't', 'a', 'm', 'i', 'a']
[ print(x,end='-') for x in lista ]
print()

#Generar una nueva lista con los elementos de la anterior
nuevaListaAleternado = [ lista[i] for i in range(0,len(lista),2) ]
print(nuevaListaAleternado)

#Nueva lista en mayúsculas
listaMayusculas = [ letra.upper() for letra in lista ]
print(listaMayusculas)

#Lista de vocales de la lista ingresada (condicional) (sin repeticiones)
vocalesLista = set([ letra for letra in lista if letra in {'a','e','i','o','u'} ])
print(vocalesLista)

#Requerimiento Héctor: diferencia de listas (conjuntos)
lista1 = ['a',2,'b','c','i','u',12,33,'perro','3',(12,55)]
lista2 = ['a',2,(12,55),'yate',33]
diferenciaListas = [ e for e in lista1 if e not in lista2 ]
print(diferenciaListas)

#Comprensión de listas -> Diccionario
#Convertir la cadena a un diccionario donde esta constituye los valores
#Forma convencional (no pythonista)
# cadena = '12 patos'
# diccionarioValores = dict()
# for i,letra in enumerate(cadena):
#     diccionarioValores[ 'Etiqueta_'+str(i) ] = letra
# print(diccionarioValores)
cadena = '12 patos'
diccionario = { 'Etiqueta_'+str(i):letra for i,letra in enumerate(cadena) }
print(diccionario)
diccionario = { letra:'Etiqueta_'+str(i) for i,letra in enumerate(cadena) }
print(diccionario)

#En una sola línea (máximo 2) -> solicitar n números al usuario y retornar
#la lista de los números que son múltiplos o de 5 o de 3. Pendiente promedio de los números

#Conversión a diccionario teniendo colecciones
llaves = ['a',2,'b','c','i','u',12,33,'perro','3',(12,55)]
valores = ['a',2,(12,55),'yate',33]
"""coleccionParejas = list(zip(llaves,valores))
print()
print()
print(coleccionParejas)"""
diccionario = dict(zip(llaves,valores))
print(diccionario)

cadena = 'mesopotamia'
diccionario = dict(zip(range(len(cadena)),cadena))
print(diccionario)


import tkinter as tk
#--↓主視窗↓--#
window = tk.Tk()                    # 建立主視窗
window.geometry('640x480')          # 設定尺寸為 640x480
window.title('YouTube 極速下載器')   # 設定主視窗標題
#--↓ Frame ↓--#
input_fm = tk.Frame(window, bg='red',   # 建立 Frame
                    width=640, height=120)
input_fm.pack() # 設定排版
#--↓ Label ↓--#
lb = tk.Label(input_fm,     # 建立 Label
              bg='red', fg='white',
              text='請輸入 YouTube 網址',
              font=('細明體', 30), padx=50, pady=80)
lb.pack()   # 設定排版
#--↓啟動主視窗↓--#
window.mainloop()