import pygame
from pygame.locals import *

from EventManager import *
from KeyboardMouseRef import *
from XboxContRef import *

class KeyboardMouseGameContController:
	"""KeyboardController takes Pygame events and notifies the control mapper
	"""
	def __init__(self, evManager):
		self.evManager = evManager
		#List every event for which this object listens
		self.evManager.registerListener(self,[TickEvent])

		pygame.joystick.init()
		#print("JOYSTICK COUNT ", pygame.joystick.get_count())
		gamepads = []
		for gamepadNumber in range (pygame.joystick.get_count()):
			newGamepad = pygame.joystick.Joystick(gamepadNumber)
			# try:
				# newGamepad = pygame.joystick.Joystick(gamepadNumber)
			# except pygame.error as message:
				# raise ErrorType(message)
			newGamepad.init()
			gamepads += [newGamepad]

		# if gamepads:
			# evManager.post()

		# joystickNo = 0
		# joy = pygame.joystick.Joystick(joystickNo)
		# joy.init()

	#----------------------------------------------------------------------
	def notify(self, event):
		if event.is_a(TickEvent ):
			#Handle Input Events
			for event in pygame.event.get():
				ev = None
				if event.type == QUIT:
					ev = QuitEvent()
				elif event.type == KEYDOWN and event.key == K_ESCAPE:
					ev = QuitEvent()
				elif event.type == KEYDOWN:
					ev = KeyboardInputEvent(event.key, 1)
				elif event.type == KEYUP:
					ev = KeyboardInputEvent(event.key, 0)
				elif event.type == MOUSEBUTTONDOWN:
					ev = MouseInputEvent(event.button, event.pos)
					#TOTRY pygame.mouse.get_pos()
				elif event.type == MOUSEBUTTONUP:
					ev = MouseInputEvent(event.button, event.pos)
				elif event.type == JOYAXISMOTION:
					#event.joy tells us which player
					if event.axis in xboxContRef.pygameAxisInt_xboxInt:
						ev = GameContInputEvent(
							event.joy,
							xboxContRef.pygameAxisInt_xboxInt[event.axis],
							event.value)
				elif event.type == JOYHATMOTION:
					ev = GameContInputEvent(event.joy, 16, event.value)
					#TOTRY event.hat I think this should always be synonomous with event.joy. Or it's for joypads with multiple dpads
				elif event.type == JOYBUTTONUP:
					ev = GameContInputEvent(event.joy, xboxContRef.pygameButtonInt_xboxInt[event.button], 0)
				elif event.type == JOYBUTTONDOWN:
					ev = GameContInputEvent(event.joy, xboxContRef.pygameButtonInt_xboxInt[event.button], 1)

				if ev:
					self.evManager.post(ev)


"""
给定一个包含红色、白色和蓝色，一共 n 个元素的数组，原地对它们进行排序，使得相同颜色的元素相邻，并按照红色、白色、蓝色顺序排列。
此题中，我们使用整数 0、 1 和 2 分别表示红色、白色和蓝色。
注意:
不能使用代码库中的排序函数来解决这道题。
示例:
输入: [2,0,2,1,1,0]
输出: [0,0,1,1,2,2]

作者：力扣 (LeetCode)
链接：https://leetcode-cn.com/leetbook/read/all-about-array/x9wv2h/
来源：力扣（LeetCode）
著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。
"""
from typing import List

class Solution:
    def sortColors(self, nums: List[int]) -> None:
        """
        Do not return anything, modify nums in-place instead.
        """
        n = len(nums)
        if n < 2:
            return
        for i in range(n):
            flag = False
            for j in range(n - 1 - i):
                if nums[j] > nums[j+1]:
                    tmp = nums[j]
                    nums[j] = nums[j+1]
                    nums[j+1] = tmp
                    flag = True
            if not flag:
                break

s = Solution()
test_n = [
    [2, 0, 2, 1, 1, 0],
]
for n in test_n:
    s.sortColors(n)
    print(n)


from django.contrib import admin
from .models import Tutorial, TutorialCategory, TutorialSeries
from tinymce.widgets import TinyMCE
from django.db import models

class TutorialAdmin(admin.ModelAdmin):
    # отображение полей при редактировании или добавлении
    # fields = ['tutorial_title', 'tutorial_content', 'tutorial_published']

    # Отображение полей секциями
    fieldsets = [
        ('Title/date/slug', {'fields': ['tutorial_title', 'tutorial_published', 'tutorial_slug', 'tutorial_series']}),
        ('Content', {'fields': ['tutorial_content']})
    ]

    formfield_overrides = {
        models.TextField: {'widget': TinyMCE()}
    }

admin.site.register(Tutorial, TutorialAdmin)
admin.site.register(TutorialCategory)
admin.site.register(TutorialSeries)


import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
import math
import scipy.stats
from mpl_toolkits.mplot3d import Axes3D
from multiprocessing import Pool
import random
import glob
import os

cpath = "/afs/.ir.stanford.edu/users/a/r/arider/gas_sim_data/s_100mum"

def build_arr(fs):
    arr = np.array([])
    for f in fs:
        arr = np.append(arr, np.loadtxt(f))
    return arr

def get_data(path):
    xfs = glob.glob(os.path.join(path, "*x.txt"))
    yfs = glob.glob(os.path.join(path, "*y.txt"))
    zfs = glob.glob(os.path.join(path, "*z.txt"))
    return build_arr(xfs),  build_arr(yfs),  build_arr(zfs)

class Fstats:
    def __init__(self, xa, ya, za, n):
        self.fs = np.array([xa, ya, za])
        self.n = n
        self.nsim = len(xa)

def gauss(x, *p):
    #a gaussian function for fitting to distributuions
    A, mu, sigma = p
    return A*np.exp(-(x-mu)**2/(2.*sigma**2))

def fit_fdist(fsobj, axis = 0, npb = 10., p0 = [100.,0., 0.00002]):
    #plots histogram of average force during simulation.
    nbins = int(len(fsobj.fs[axis, :])/npb)
    h, be = np.histogram(fsobj.fs[axis, :], nbins)
    bc = (be[1:] + be[:-1])/2.
    popt, pcov = curve_fit(gauss, bc, h, p0 = p0)
    mustr = '{:.2e}'.format(popt[1])
    sigstr = '{:.2e}'.format(np.sqrt(pcov[1, 1]))
    pltx = np.linspace(np.min(bc), np.max(bc), 100)
    plt.plot(pltx, gauss(pltx, *popt), label = \
             'mean ' + mustr+" $\pm$ " +sigstr, linewidth = 2.)
    plt.hist(fsobj.fs[axis, :], nbins)
    plt.xlabel('Force after %i collisions'%fsobj.nsim)
    plt.ylabel('Frequency in %i simulations'%fsobj.n)
    plt.legend()
    plt.show()
    return pcov

def plter(xarr, yarr, zarr):
    #plots the trajectories
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot(xarr, yarr, zarr, '.-')
    ax.set_xlabel('x')
    ax.set_ylabel('y')
    ax.set_zlabel('z')
    #ax.set_xlim([-1, 1])
    #ax.set_ylim([-1, 1])
    #ax.set_zlim([-1, 1])
    plt.show()

xarr, yarr, zarr = get_data(cpath)
fsobj = Fstats(xarr, yarr, zarr, 10000000)
fit_fdist(fsobj, axis = 1)

#plter(xarr, yarr, zarr)


try:
    from PIL import Image
except ImportError:
    import Image
import pytesseract
import cv2

file = 'sample.jpg'

img = cv2.imread(file, cv2.IMREAD_GRAYSCALE)
img = cv2.resize(img, None, fx=10, fy=10, interpolation=cv2.INTER_LINEAR)
img = cv2.medianBlur(img, 9)
th, img = cv2.threshold(img, 185, 255, cv2.THRESH_BINARY)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (4,8))
img = cv2.morphologyEx(img, cv2.MORPH_CLOSE, kernel)
cv2.imwrite("sample2.jpg", img)

file = 'sample2.jpg'
text = pytesseract.image_to_string(file)
print(''.join(x for x in text if x.isdigit()))


class Solution:
    def permute(self, nums):
        """
        给出一个没有重复数字的序列的全排列
        :param nums: 五重复数字的序列
        :return: 全排列表
        """
        if len(nums) <= 1:
            return [nums]

        answer_list = []

        answer = []
        flag = [False for i in range(0, len(nums))]

        def _find():
            """
            递归查找所有的排列
            :return:
            """
            if len(answer) == len(nums):
                answer_list.append(answer.copy())
                return

            for i in range(0, len(nums)):
                if flag[i] is False:
                    answer.append(nums[i])
                    flag[i] = True
                    _find()
                    answer.remove(nums[i])
                    flag[i] = False

        _find()
        return answer_list

if __name__ == "__main__":
    print(Solution().permute([1, 2, 3, 4]))


"""
    Create by: apenasrr
    Source: https://github.com/apenasrr/mass_videojoin
"""

import os
import pandas as pd
import datetime
import logging
from video_tools import change_width_height_mp4, get_video_details, \
                        join_mp4, split_mp4
from config_handler import handle_config_file
import unidecode
import natsort
import glob

def logging_config():

    logfilename = 'log-' + 'mass_videojoin' + '.txt'
    logging.basicConfig(filename=logfilename, level=logging.DEBUG,
                        format=' %(asctime)s-%(levelname)s-%(message)s')
    # set up logging to console
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    # set a format which is simpler for console use
    formatter = logging.Formatter(' %(asctime)s-%(levelname)s-%(message)s')
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger('').addHandler(console)
    logger = logging.getLogger(__name__)

def clean_cmd():

    clear = lambda: os.system('cls')
    clear()

def df_sort_human(df):
    """
    Sort files and folders in human way.
    So after folder/file '1' comes '2', instead of '10' and '11'.
    Simple yet flexible natural sorting in Python.
    When you try to sort a list of strings that contain numbers,
    the normal python sort algorithm sorts lexicographically,
    so you might not get the results that you expect:
    More info: https://github.com/SethMMorton/natsort

    :input: DataFrame. With columns [file_folder, file_name]
    :return: DataFrame. Sort in a human way by [file_folder, file_name]
    """

    def sort_human(list_):

        list_ = natsort.natsorted(list_)

        return list_

    def sort_df_column_from_list(df, column_name, sorter):
        """
        :input: df: DataFrame
        :input: column_name: String
        :input: sorter: List
        :return: DataFrame
        """

        sorterIndex = dict(zip(sorter, range(len(sorter))))
        df['order'] = df[column_name].map(sorterIndex)
        df.sort_values(['order'], ascending = [True ], inplace = True)
        df.drop(['order', column_name], 1, inplace = True)

        return df

    column_name = 'path_file'
    df[column_name] = df['file_folder'] + '\\' + df['file_name']
    list_path_file = df[column_name].tolist()
    sorter = sort_human(list_path_file)
    df = sort_df_column_from_list(df, column_name, sorter)

    return df

def gen_report(path_dir):

    # TODO input more file video extension:
    ## https://dotwhat.net/type/video-movie-files

    l=[]
    for root, dirs, files in os.walk(path_dir):
        for file in files:
            file_lower = file.lower()
            if file_lower.endswith((".mp4", ".avi", ".webm", '.ts', '.vob',
                                    '.mov')):
                print(file)

                path_file = os.path.join(root, file)
                dict_inf = get_video_details(path_file)
                (mode, ino, dev, nlink, uid,
                gid, size, atime, mtime, ctime) = os.stat(path_file)
                mtime = datetime.datetime.fromtimestamp(mtime)
                d={}
                d['mtime']=mtime
                d['file_folder'] = root
                d['file_name'] = file
                d['file_size'] = os.path.getsize(path_file)

                try:
                    d['duration'] = dict_inf['duration']
                except:
                    logging.error('video without duration: {path_file}')
                    d['duration'] = ''

                d['bitrate'] = dict_inf['bitrate']
                d['video_codec'] = dict_inf['video']['codec']
                d['video_profile'] = dict_inf['video']['profile']
                d['video_resolution'] = dict_inf['video']['resolution']
                d['video_bitrate'] = dict_inf['video']['bitrate']

                # some videos dont have audio
                try:
                    d['audio_codec'] = dict_inf['audio']['codec']
                    d['audio_frequency'] = dict_inf['audio']['frequency']
                    d['audio_bitrate'] = dict_inf['audio']['bitrate']
                except:
                    d['audio_codec'] = ''
                    d['audio_frequency'] = ''
                    d['audio_bitrate'] = ''
                d['video_resolution_to_change'] = ''
                l.append(d)
    df = pd.DataFrame(l)

    return df

def get_video_details_with_group(df):

    df['key_join_checker'] = df['audio_codec'] + '-' + \
                             df['video_codec'] + '-' + \
                             df['video_resolution']

    # set group_encode
    df['group_encode'] = 1
    for index, row in df.iterrows():
        if index>0:
            group_encode_value_prev = df.loc[index-1, 'group_encode']
            if row['key_join_checker'] != df.loc[index-1, 'key_join_checker']:
                df.loc[index, 'group_encode'] = group_encode_value_prev + 1
            else:
                df.loc[index, 'group_encode'] = group_encode_value_prev

    return df

def get_list_chunk_videos_from_group(df, group_no, max_size_mb):

    max_size_bytes = max_size_mb * 1024**2
    mask = df['group_encode'].isin([group_no])

    df['file_path'] = df['file_folder'] + '\\' + \
                      df['file_name']

    df_group = df.loc[mask, :]
    list_chunk_videos = []
    chunk_size = 0
    list_videos = []
    for index, row in df_group.iterrows():
        if chunk_size + row['file_size'] > max_size_bytes:
            logging.info(f'join video from {len(list_videos)} files')
            list_chunk_videos.append(list_videos)

            list_videos = []
            chunk_size = 0

        list_videos.append(row['file_path'])
        chunk_size += row['file_size']

    if len(list_videos) > 0:
        logging.info(f'join video from {len(list_videos)} files')
        list_chunk_videos.append(list_videos)
        list_videos = []

    logging.info(f'group {group_no} will generate ' + \
                 f'{len(list_chunk_videos)} videos')
    return list_chunk_videos

def get_list_chunk_videos(df, max_size_mb):

    list_group = df['group_encode'].unique().tolist()
    list_final = []

    for group_no in list_group:
        group_no = str(group_no)
        list_chunk_videos = get_list_chunk_videos_from_group(df, group_no,
                                                             max_size_mb)
        list_final += list_chunk_videos
        print('')

    return list_final

def get_name_dir_origin():

    name_file_folder_name = get_txt_folder_origin()
    dir_name_saved = get_txt_content(name_file_folder_name)

    return dir_name_saved

def get_path_folder_output_video():

    path_folder_output = get_name_dir_origin()
    folder_name = 'output_' + path_folder_output
    ensure_folder_existence([folder_name])

    path_folder_output_video = os.path.join(folder_name, 'output_videos')
    ensure_folder_existence([path_folder_output_video])

    return path_folder_output_video

def join_videos(df, max_size_mb, start_index_output):

    # path_folder_output = userpref_folderoutput()
    path_folder_output = get_path_folder_output_video()

    # default_filename_output = input('Enter a default name for the joined ' +\
                                    # 'videos: ')
    default_filename_output = get_name_dir_origin()

    df['file_path'] = df['file_folder'] + '\\' + df['file_name']
    list_chunk_videos = get_list_chunk_videos(df, max_size_mb)
    df['file_output'] = ''

    for index, list_file_path in enumerate(list_chunk_videos):
        file_count = index + start_index_output
        file_name_output = f'{default_filename_output}-%03d.mp4' % file_count
        file_path_output = os.path.join(path_folder_output, file_name_output)
        join_mp4(list_file_path, file_path_output)

        # register file_output in dataframe
        mask_files_joined = df['file_path'].isin(list_file_path)
        df.loc[mask_files_joined, 'file_output'] = file_name_output

    print(f'total: {len(list_chunk_videos)} videos')

    return df

def exclude_all_files_from_folder(path_folder):

        path_folder_regex = os.path.join(path_folder, '*')
        r = glob.glob(path_folder_regex)
        for i in r:
            os.remove(i)

def make_reencode(df):

    folder_script_path = get_folder_script_path()
    path_folder_encoded = os.path.join(folder_script_path, 'videos_encoded')
    exclude_all_files_from_folder(path_folder_encoded)

    # exclude_folder_videos_encoded()

    df['file_folder_origin'] = df['file_folder']
    df['file_name_origin'] = df['file_name']
    df['file_size_origin'] = df['file_size']
    df['video_resolution_origin'] = df['video_resolution']
    mask_df_to_reencode = ~df['video_resolution_to_change'].isna()
    df_to_reencode = df.loc[mask_df_to_reencode, :]

    folder_script_path = get_folder_script_path()
    path_folder_dest = os.path.join(folder_script_path, 'videos_encoded')
    for index, row in df_to_reencode.iterrows():
        size_width, size_height = row['video_resolution_to_change'].split('x')

        path_file_origin = os.path.join(row['file_folder_origin'],
                                        row['file_name_origin'])
        print(path_file_origin)
        # path_folder_dest = r'videos_encoded'
        path_file_name_dest = str(index) + '.mp4'
        path_file_dest = os.path.join(path_folder_dest,
                                      path_file_name_dest)

        # todo reencode
        # input path_folder_dest in column file_folder
        df.loc[index, 'file_folder'] = os.path.abspath(path_folder_dest)
        # input path_file_name_dest in column file_name
        df.loc[index, 'file_name'] = path_file_name_dest

        change_width_height_mp4(path_file_origin, size_height,
                                size_width, path_file_dest)

        file_size = os.stat(path_file_dest).st_size
        df.loc[index, 'file_size'] = file_size
        df.loc[index, 'video_resolution'] = row['video_resolution_to_change']

        # from encoded video get video metadata
        metadata = get_video_details(path_file_dest)
        # register video metadata
        df.loc[index, 'bitrate'] = metadata['bitrate']
        df.loc[index, 'video_bitrate'] = metadata['video']['bitrate']
        df.loc[index, 'video_codec'] = metadata['video']['codec']
        df.loc[index, 'audio_codec'] = metadata['audio']['codec']
        df.loc[index, 'audio_bitrate']  = metadata['audio']['bitrate']
        df.loc[index, 'duration'] = metadata['duration']

    return df

def menu_ask():

    # ptbr
    # print('1-Gerar planilha listando os arquivos')
    # print('2-Processar reencode dos vídeos marcados na coluna '+
          # '"video_resolution_to_change"')
    # print('3-Agrupar vídeos em grupos de até 1 gb com mesmo codec e resolução')

    # eng
    print('1-Generate worksheet listing the files')
    print('2-Process reencode of videos marked in column ' +
          '"video_resolution_to_change"')
    print('3-Group videos into groups up to 1 gb with the same codec ' + \
          'and resolution')

    # ptbr
    # msg_type_answer = 'Digite sua resposta: '

    # eng
    msg_type_answer = 'Type your answer: '
    make_report = int(input(f'\n{msg_type_answer}'))
    if make_report == 1:
        return 1
    elif make_report == 2:
        return 2
    elif make_report == 3:
        return 3
    else:
        # ptbr
        # msg_invalid_option = "Opção não disponível"

        # eng
        msg_invalid_option = "Invalid option"
        raise MyValidationError(msg_invalid_option)

def df_insert_row(row_number, df, row_value):
    """
    A customized function to insert a row at any given position in the
     dataframe.
    source: https://www.geeksforgeeks.org/insert-row-at-given-position-in-pandas-dataframe/
    :input: row_number: Int.
    :input: df: Dataframe.
    :input: row_value: Int.
    :return: Dataframe. df_result |
             Boolean. False. If the row_number was invalid.
    """

    if row_number > df.index.max()+1:
        print("df_insert_row: Invalid row_number")
        return False

    # Slice the upper half of the dataframe
    df1 = df[0:row_number]

    # Store the result of lower half of the dataframe
    df2 = df[row_number:]

    # Inser the row in the upper half dataframe
    df1.loc[row_number]=row_value

    # Concat the two dataframes
    df_result = pd.concat([df1, df2])

    # Reassign the index labels
    df_result.index = [*range(df_result.shape[0])]

    # Return the updated dataframe
    return df_result

def search_to_split_videos(df, mb_limit):

    def preprocess_df_split(df):

        df['file_path'] = df['file_folder'] + '\\' + df['file_name']
        df['split_file_folder_origin'] = ''
        df['split_file_name_origin'] = ''
        df['split_file_size_origin'] = ''

        return df

    def get_dict_row_dest(dict_row_origin, pathfile_output):

        path_folder_dest = os.path.split(pathfile_output)[0]
        file_name_dest = os.path.split(pathfile_output)[1]
        file_size_dest = os.stat(pathfile_output).st_size

        dict_row_dest = dict_row_origin.copy()
        dict_row_dest['split_file_folder_origin'] = \
            dict_row_origin['file_folder']
        dict_row_dest['split_file_name_origin'] = dict_row_origin['file_name']
        dict_row_dest['split_file_size_origin'] = dict_row_origin['file_size']

        dict_row_dest['file_folder'] = path_folder_dest
        dict_row_dest['file_name'] = file_name_dest
        dict_row_dest['file_size'] = file_size_dest

        return dict_row_dest

    def get_row_number_from_filepath(df, file_path_origin):

        path_folder_origin = os.path.split(file_path_origin)[0]
        file_name_origin = os.path.split(file_path_origin)[1]

        mask1 = df['file_folder'].isin([path_folder_origin])
        mask2 = df['file_name'].isin([file_name_origin])
        mask_file = mask1 & mask2

        df_row_origin = df.loc[mask_file, :]
        dict_row_origin = df_row_origin.iloc[0]
        row_number = df_row_origin.index.values[0]

        return row_number

    def delete_fileorigin(df, file_path_origin):

        row_number = get_row_number_from_filepath(df, file_path_origin)
        df = df.drop(df.index[[row_number]])
        df = df.reset_index(drop=True)
        return df

    def update_df_files(df, file_path_origin, list_filepath_output):

        def include_rows_new_files(df, filepath_output):

            pathfile_output = os.path.abspath(filepath_output)
            # find the row_number of origin file
            row_number = get_row_number_from_filepath(df, file_path_origin)
            dict_row_origin = df.loc[row_number,:]
            dict_row_dest = get_dict_row_dest(dict_row_origin, pathfile_output)
            df = df_insert_row(row_number=row_number, df=df,
                                   row_value=dict_row_dest)
            return df

        for filepath_output in list_filepath_output:
            # include the rows corresponding to the new files created
            df = include_rows_new_files(df=df,
                                        filepath_output=filepath_output)

        # delete the file origin row
        df = delete_fileorigin(df=df, file_path_origin=file_path_origin)

        return df

    def get_list_file_path_origin(df, size_limit):

        mask_to_be_split = df['file_size'] > size_limit
        df_to_be_split = df.loc[mask_to_be_split, :]
        list_file_path_origin = df_to_be_split.loc[:, 'file_path'].tolist()

        return list_file_path_origin

    df = preprocess_df_split(df)

    recoil_sec = 10
    # TODO estimate the recoil_mbsize by video bitrate
    recoil_mbsize = 10
    size_limit = (mb_limit-recoil_mbsize) * 1024**2
    list_file_path_origin = get_list_file_path_origin(df=df,
                                                      size_limit=size_limit)
    folder_script_path = get_folder_script_path()
    output_folder_path = os.path.join(folder_script_path, 'videos_splitted')
    exclude_all_files_from_folder(output_folder_path)

    for file_path_origin in list_file_path_origin:
        list_filepath_output = split_mp4(largefile_path=file_path_origin,
                                         recoil=recoil_sec,
                                         mb_limit=mb_limit,
                                         output_folder_path=output_folder_path)

        df = update_df_files(df=df, file_path_origin=file_path_origin,
                             list_filepath_output=list_filepath_output)

    return df

def userpref_folderoutput():

    path_file = os.path.join('config', 'config.txt')
    variable_name = 'path_folder_output'
    variable_value = \
        handle_config_file(path_file, variable_name,
                           set_value=None, parse=True)
    path_folder_output = variable_value['path_folder_output'][0]

    print(f'Use the folder path output as {path_folder_output}?')
    answer_use = input('(None for yes) Answer: ')
    if answer_use == '':
        pass
    else:
        path_folder_output = input('Inform the folder path output: ')
        handle_config_file(path_file, variable_name,
                           set_value=path_folder_output, parse=False)

    return path_folder_output

def userpref_size_per_file_mb():

    folder_script_path = get_folder_script_path()
    path_file = os.path.join(folder_script_path, 'config', 'config.txt')
    variable_name = 'size_per_file_mb'

    question = 'What should be the maximum size of each ' + \
               'file in mb (e.g.: 500)? '

    variable_value = \
        handle_config_file(path_file, variable_name,
                           set_value=None, parse=True)
    value_got = variable_value[variable_name][0]

    print(f'The maximum size of each file will be {value_got}. Ok?')
    answer_use = input('(None for yes) Answer: ')
    if answer_use == '':
        pass
    else:
        value_got = input(question)
        handle_config_file(path_file, variable_name,
                           set_value=value_got, parse=False)

    return value_got

def ensure_folder_existence(folders_path):
    """
    :input: folders_path: List
    """

    for folder_path in folders_path:
        existence = os.path.isdir(folder_path)
        if existence is False:
            os.mkdir(folder_path)

def ensure_folders_existence():

    folder_script_path_relative = os.path.dirname(__file__)
    folder_script_path = os.path.realpath(folder_script_path_relative)

    folders_name = ['ts', 'videos_encoded', 'config', 'videos_splitted']
    folders_path = []
    for folder_name in folders_name:
        folder_path = os.path.join(folder_script_path, folder_name)
        folders_path.append(folder_path)

    ensure_folder_existence(folders_path)

def get_txt_content(file_path):

    file = open(file_path,'r', encoding='utf-8')
    file_content = file.readlines()
    file_content = ''.join(file_content)
    file.close()

    return file_content

def create_txt(file_path, stringa):

    f = open(file_path, "w", encoding='utf8')
    f.write(stringa)
    f.close()

def get_folder_name_normalized(path_dir):

    def normalize_string_to_link(string_actual):

        string_new = unidecode.unidecode(string_actual)

        for c in r"!@#$%^&*()[]{};:,./<>?\|`~-=_+":
            string_new = string_new.translate({ord(c): "_"})

        string_new = string_new.replace(' ', '_')
        string_new = string_new.replace('___', '_')
        string_new = string_new.replace('__', '_')

        return string_new

    dir_name = os.path.basename(path_dir)
    dir_name_normalize = normalize_string_to_link(dir_name)

    return dir_name_normalize

def prefill_video_resolution_to_change(df):

    df['key_join_checker'] = df['audio_codec'] + '-' + \
                             df['video_codec'] + '-' + \
                             df['video_resolution']

    df_key = df[['key_join_checker', 'duration', 'video_resolution']].copy()
    df_key['duration_timedelta'] = pd.to_timedelta(df_key['duration'])
    df_key['duration_min'] = df_key['duration_timedelta'].dt.total_seconds()/60

    df_key.drop('duration', axis=1, inplace=True)

    df_key_agg = df_key.groupby(['key_join_checker',
                                 'video_resolution'])['duration_min'].agg('sum')
    # convert in dataframe
    df_key_agg = df_key_agg.reset_index()
    # sort dataframe
    df_key_agg = df_key_agg.sort_values(['duration_min'], ascending=[False])

    print('\n', df_key_agg)
    index_max = df_key_agg['duration_min'].idxmax()
    key_join_main = df_key_agg.loc[index_max, 'key_join_checker']
    video_resolution_main = df_key_agg.loc[index_max, 'video_resolution']

    # informar quantidade de minutos para reencodar
    mask_to_convert = ~df_key_agg.index.isin([index_max])
    minutes_to_reencode = df_key_agg.loc[mask_to_convert, 'duration_min'].sum()
    minutes_total = df_key_agg['duration_min'].sum()
    percent_to_reencode = minutes_to_reencode/minutes_total
    print(f'The main profile is "{key_join_main}"')

    # informar percentual de minutos para reencodar
    # the command ':.1f' fix 1 digit after decimal point
    print(f'There is {minutes_to_reencode:.1f} minutes ' + \
          f'({percent_to_reencode*100:.0f}%) to reencode')

    mask_resolution_to_change = ~df['key_join_checker'].isin([key_join_main])
    df.loc[mask_resolution_to_change,
           'video_resolution_to_change'] = video_resolution_main
    df.drop('key_join_checker', axis=1, inplace=True)

    return df

def save_upload_folder_name(path_dir, file_folder_name):

    dir_name_normalize = get_folder_name_normalized(path_dir)
    create_txt(file_path=file_folder_name, stringa=dir_name_normalize)

def create_report_backup(df, path_file_report, tag):

    path_folder = os.path.dirname(path_file_report)
    file_name = os.path.basename(path_file_report)
    file_name_without_extension = os.path.splitext(file_name)[0]
    file_name_backup = file_name_without_extension + "_" + tag + ".xlsx"
    path_file_backup = os.path.join(path_folder, file_name_backup)
    df.to_excel(path_file_backup, index=False)

def step_create_report_filled(path_dir, path_file_report):

    df = gen_report(path_dir)
    # sort path_file by natural human way
    df = df_sort_human(df)
    # prefill column video_resolution_to_change
    df = prefill_video_resolution_to_change(df)
    df.to_excel(path_file_report, index=False)

    #Make backup
    create_report_backup(df=df, path_file_report=path_file_report, tag='origin')

def set_make_reencode(path_file_report):

    df = pd.read_excel(path_file_report)
    df = make_reencode(df)
    df.to_excel(path_file_report, index=False)

    #make backup
    create_report_backup(df=df, path_file_report=path_file_report, tag='reencode')
    print('\nReencode finished')

def set_group_column(path_file_report):

    # update video_details with groups

    df = pd.read_excel(path_file_report)
    df = get_video_details_with_group(df)
    df.to_excel(path_file_report, index=False)
    print(f"File '{path_file_report}' was updated with " + \
          f"group column to fast join\n")
    # backup is not performed here as the grouping can be adjusted manually

def set_split_videos(path_file_report, mb_limit):

    df = pd.read_excel(path_file_report)

    #backup group, after adjusted manually
    create_report_backup(df=df, path_file_report=path_file_report, tag='grouped')

    # Find for file_video too big and split them
    df = search_to_split_videos(df, mb_limit=mb_limit)

    df.to_excel(path_file_report, index=False)

    create_report_backup(df=df, path_file_report=path_file_report, tag='splited')

def set_join_videos(path_file_report, mb_limit, start_index_output):

    df = pd.read_excel(path_file_report)
    df = join_videos(df, mb_limit, start_index_output)
    df.to_excel(path_file_report, index=False)

    #backup joined
    create_report_backup(df=df, path_file_report=path_file_report, tag='joined')

def get_folder_script_path():

    folder_script_path_relative = os.path.dirname(__file__)
    folder_script_path = os.path.realpath(folder_script_path_relative)

    return folder_script_path

def set_path_file_report():

    folder_path_output_relative = 'output_' + get_name_dir_origin()
    ensure_folder_existence([folder_path_output_relative])
    path_file_report = os.path.join(folder_path_output_relative,
                                    'video_details.xlsx')

    return path_file_report

def get_txt_folder_origin():

    file_folder_name = 'folder_files_origin.txt'

    return file_folder_name

def get_start_index_output():

    print('Start output file count with what value?')
    add_num = input('(None for 1) Answer: ')
    if add_num == '':
        add_num = 1
    else:
        add_num = int(add_num)
    return add_num

def main():

    ensure_folders_existence()
    file_name_folder_origin = get_txt_folder_origin()

    path_file_report = set_path_file_report()

    menu_answer = menu_ask()

    if menu_answer == 1:
        # create Dataframe of video details
        path_dir = input('\nPaste the folder link where are the video files: ')

        # save in txt, the folder name
        save_upload_folder_name(path_dir, file_name_folder_origin)

        path_file_report = set_path_file_report()

        step_create_report_filled(path_dir, path_file_report)

        print(f'\nIf necessary, change the reencode plan in the column ' + \
              f'"video_resolution_to_change"')

        break_point = input('Type Enter to continue')
        clean_cmd()
        main()
        return

    elif menu_answer == 2:

        # reencode videos mark in column video_resolution_to_change
        set_make_reencode(path_file_report)

        break_point = input('Review the file and then type something to ' + \
                            'continue.')
        clean_cmd()
        main()
        return

    elif menu_answer == 3:

        mb_limit = int(userpref_size_per_file_mb())

        # establishes separation criteria for the join videos step
        set_group_column(path_file_report)

        break_point = input('Review the file and then type something to ' + \
                            'start the process that look for videos that ' + \
                            'are too big and should be splitted')

        set_split_videos(path_file_report, mb_limit)

        # set start index output file
        start_index_output = get_start_index_output()

        # join all videos
        set_join_videos(path_file_report, mb_limit, start_index_output)
        return
    else:

        return

if __name__ == "__main__":
    logging_config()
    main()


import socket
import ssl
import sys

# Color
green =  '\033[32m' # Green color
nocolor = '\033[m'  # Nocolor

# Function to query cloudfare server through an encrypted socket and return output from cloudflare
def querydns(socktcp_encrypted,data):
  tcp_query = ("\x00"+chr(len(data)) + data)
  socktcp_encrypted.send(tcp_query)
  outputfromdns=socktcp_encrypted.recv(1024)
  return outputfromdns

# Function to create a TCP socket and wrap it with TLS
def tcp_conn_encrypted(dnsserver):
  socktcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  socktcp.settimeout(10)
  context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
  context.verify_mode = ssl.CERT_REQUIRED
  context.load_verify_locations('/etc/ssl/certs/ca-certificates.crt')
  socktcp_wrapped = context.wrap_socket(socktcp, server_hostname=dnsserver)
  socktcp_wrapped.connect((dnsserver, 853))
  print(green + ' \n*********** Certificate details from Cloudflare DNS server ***********' + nocolor)
  print(socktcp_wrapped.getpeercert())
  print(green + '************************* END of Certificate ************************* \n' + nocolor)
  return socktcp_wrapped


# -*- coding: utf-8 -*-
import base64
from django.conf import settings
from cryptography.fernet import Fernet

cipher = Fernet(base64.urlsafe_b64encode(settings.SECRET_KEY.encode()[0:32]).decode())

def encrypt(text, b16=False):
    s = cipher.encrypt(str(text).encode('utf-8'))
    if b16:
        s = base64.b16encode(s)
    return s.decode('utf-8')

def decrypt(text, b16=False):
    if b16:
        text = base64.b16decode(text.encode('utf-8'))
    else:
        text = text.encode('utf-8')
    s = cipher.decrypt(text)
    return s.decode('utf-8')


# coding: utf-8

from django.shortcuts import render
from django.shortcuts import redirect
from django.http import HttpResponse, Http404
from django.template import loader
#from django.core.exceptions import ObjectDoesNotExist

from .forms import *
from .models import *
from django.contrib.auth.models import*
from django.contrib.auth import *
from bs4 import BeautifulSoup
from django.template.response import *
from datetime import *
import bs4
import re
import pickle
import os
from nltk import *
from bs4 import BeautifulSoup
from random import *
from .creation_graph import *
from unidecode import unidecode
from django.utils.safestring import mark_safe

app_name = 'revendication'

#_____________________vue_______________________#

def page_accueil(request):
	if request.user.is_authenticated:
		pass
	else:
		return render(request, 'revendications/varthing.html')
	from django.contrib.auth.models import User
	utilisateur = request.user

	def creer_les_evenements_du_calendriers ():

		evenements = Evenement.objects.all()
		fichier_evenement = u""
		for evenement in evenements:
			ligne =  evenement.titre + "/" + str(evenement.date) + "/" + str(evenement.id) + "/" + str(evenement.description) +"ggg"
			fichier_evenement = fichier_evenement + ligne

		fichier_evenement = unidecode(fichier_evenement).encode("utf-8")
		#print ("************************** fichier_evenement", fichier_evenement)
		return fichier_evenement

	def revendications():
		propositions = Proposition.objects.all()
		mes_propositions = Proposition.objects.filter(soutien__user = utilisateur, soutien__lien = "SO")
		date_M1 = datetime.now()-timedelta(30)
		liste= []
        #progression?
		for p in propositions:
			soutiens_M1 = Soutien.objects.filter(date__lte = date_M1, propositions = p)
			nb_soutiens = len(soutiens_M1)
			p.progression = nb_soutiens
			liste.append(p)
			if p in mes_propositions:
				p.mienne = "oui"
			else:
				p.mienne = "non"
		#print("liste______", liste)
		return liste

	def documents():
		documents = Document.objects.all()
		return documents

	def evenements():
		liste_de_mes_evenements = Evenement.objects.filter(soutien__user = utilisateur)
		liste_de_mes_propositions = Proposition.objects.filter(soutien__user = utilisateur)
		evenements = Evenement.objects.all()
		liste = []
		for ev in evenements:
			if ev in liste_de_mes_evenements:
				ev.mienne = "oui"
			else:
				ev.mienne = "non"
			liste.append(ev)
		return liste

	def competences():
		liste_de_mes_competences = Competence.objects.filter(soutien__user = utilisateur)
		liste_de_mes_propositions = Proposition.objects.filter(soutien__user = utilisateur)
		competences = Competence.objects.all()
		liste = []
		for comp in competences:
			if comp in liste_de_mes_competences :
				comp.mienne = "oui"
			else:
				comp.mienne = "non"
			liste.append(comp)
		return liste

	def organisations():
	#creer la liste des organisations dont je soutiens les propositions
		liste_de_mes_organisations = Organisation.objects.filter(soutien__user = utilisateur)
		liste_de_mes_propositions = Proposition.objects.filter(soutien__user = utilisateur)
		organisations = Organisation.objects.all()
		liste = []
		for org in organisations:
			if org in liste_de_mes_organisations:
				org.mienne = "oui"
			else:
				org.mienne = "non"
			liste.append(org)
		return liste

	def petitions():
		liste_de_mes_petitions = Petition.objects.filter(soutien__user = utilisateur)
		liste_de_mes_propositions = Proposition.objects.filter(soutien__user = utilisateur)
		petitions = Petition.objects.all()
		liste = []
		for p in petitions:
			if p in liste_de_mes_petitions :
				p.mienne = "oui"
			else:
				p.mienne = "non"
			liste.append(p)
		return liste

	class formulaire:
		def __init__ (self):
			self.revendication = RevendicationForm
			self.petition =PetitionForm
			self.evenement= EvenementForm

	def trier_les_elements_par_dates(liste):
		liste = sorted(liste, key=lambda x: x[1])
		liste.reverse()

	def creer_les_datas(utilisateur):

		class Datas:
			def __init__ (self):
				self.evenements = evenements()
				self.competences = competences()
				self.documents = documents()
				self.petitions = petitions()
				self.revendications = revendications()
				self.organisations = organisations()
				self.suggestions = self.revendications

				#formulaires:
				class formulaire:
					def __init__ (self):
						self.revendication = RevendicationForm
						self.petition =PetitionForm
						self.evenement= EvenementForm
				self.formulaires = formulaire()

		datas = Datas()
		return datas

	datas = creer_les_datas(utilisateur)

	try:
		onglet = request.session["onglet"]
	except:
		onglet = "vide"

	graph_a =  graph_accueil()

	request.session['message']="vide"
	request.session["proposition_id"]="toutes"

	message = request.session['message']
	return render(request, 'revendications/page_accueil.html', {"datas":datas, "graph_accueil":mark_safe(graph_a),"onglet": onglet, "message":mark_safe(message)})


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Mar  4 15:16:07 2019

@author: elimiller
"""

import numpy as np
import matplotlib.pyplot as plt
import scipy.signal
import librosa
import sklearn

#%%
def get_features(data, rate, samples, label, plot_spectrograms=False):

    data = np.array(data, dtype='float64')

    sample_length = 5 * rate
    #indiceis for 5 second clips
    num_samples = int(np.floor(len(data) / sample_length))
    #round so that we can reshape without residuals

    trunc_index = sample_length * num_samples

    data_mat = np.reshape(data[0:trunc_index], (num_samples, sample_length))
    np.random.shuffle(data_mat)

    if samples == 'max':
        samples = num_samples
    data_set = data_mat[0:samples, :]

    feature = [label] * np.shape(data_set)[0]
    #make a list of input genre for the album data
    for j in range(samples):
        f, t, Sxx =  scipy.signal.spectrogram(
                data_set[j,:], rate, scaling='spectrum', mode='magnitude')
        if plot_spectrograms:
            plt.figure()
            plt.pcolormesh(t, f, Sxx, cmap='plasma')
            plt.title(label.upper())
            plt.xlabel('Time $t$')
            plt.ylabel('Frequency $\omega$')

        if j ==0:
            spectrogram_data = np.zeros((samples, np.size(Sxx.flatten())))

        spectrogram_data[j, :] = Sxx.flatten()
    return spectrogram_data, feature

#%%
genres = np.array(['jazz', 'rock', 'edm'])
#genres = np.array(['rock'])
samplenums = np.array(['0', '1', '2'])
#samplenums = np.array(['0'])

MAKEFILES = True
SAVEFILES = False
LOADFILES = False

short_length = 60*25
#duration=short_length

#samples_per_file = int(short_length / 5)
samples_per_file = 'max'

A = []
labels = []

for genre in genres:
    for samplenum in samplenums:
        if MAKEFILES:
            path = '/Users/elimiller/Desktop/AMATH482/HW4/Audio_Files/' + genre + samplenum + '.mp3'
            data, rate = librosa.load(
                    path, res_type='kaiser_fast', offset=0, duration  = 25*60)
        if LOADFILES:
            rate = 22050
            data = np.load(genre+samplenum+'.npy')

        if SAVEFILES:
            np.save('%s%s'%(genre, samplenum),data)

        print('genre: %s, sample: %s loaded' %(genre, samplenum))

        x_data, y_data = get_features(
                data,
                rate,
                samples_per_file,
                genre,
                plot_spectrograms=False)

        A.append(x_data)
        labels.append(y_data)

A_array = np.array(A)
A_new = np.reshape(
        A_array,
        (np.shape(A_array)[0]*np.shape(A_array)[1], np.shape(A_array)[2]))

labels_array = np.array(labels)
labels_new = np.ravel(np.reshape(labels_array, (np.size(labels_array),1)))

A_new_centered = A_new - A_new.mean(axis=1, keepdims=True)
U, S, V = np.linalg.svd(A_new_centered, full_matrices =False)
plt.figure()
plt.plot(S/np.sum(S), 'o-')
plt.title('Singular Value Spectrum of Audio Data')

import warnings
warnings.filterwarnings("ignore", category=FutureWarning)
#suppress warnings for printouts
from sklearn.model_selection import train_test_split
from sklearn.svm import LinearSVC

rank = 25

representation = A_new_centered @ V.T[:, 0:rank]

x_train, x_test, y_train, y_test = train_test_split(
        representation, labels_new, train_size=.80)

svm_clf = LinearSVC()
svm_clf.fit(x_train, y_train)
#y_predict = clf.predict(x_test)
svm_score = svm_clf.score(x_test, y_test)
print('SVM Accuracy %f' %svm_score)

path = '/Users/elimiller/Desktop/AMATH482/HW4/Audio_Files/DudeLooksLikeALady.mp3'
val_data, val_rate = librosa.load(
                    path, res_type='kaiser_fast', offset=0)

validation_set, validation_labels = get_features(
                val_data,
                val_rate,
                'max',
                'rock')

validation_centered = validation_set - validation_set.mean(
        axis=1, keepdims=True)
validation_rep = validation_centered @ V.T[:, 0:rank]
val_score = svm_clf.score(validation_rep, validation_labels)
print('Validation Score: %f' %val_score)


from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords

from pymystem3 import Mystem
mystem = Mystem()

class TextProcessor:
    def __init__(self):
        self.tokenizer = RegexpTokenizer(r'[а-яА-ЯёЁ]+')
        self.stop_words = stopwords.words('russian')

    def process(self, text):
        tokenized = self.tokenizer.tokenize(text)
        lemmatized = [mystem.lemmatize(token)[0] for token in tokenized]
        without_stopwords = [word for word in lemmatized if word not in self.stop_words]

        return " ".join(without_stopwords)


import numpy as np
import networkx as nx
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cmx

"""
Plotting code used for general visualization
"""

# tie_decay_result = np.load("tie_decay_1e3.npy").item()

print ("Now start plotting")

G = nx.Graph()
tie_decay_edges = [(k[0], k[1], v) for k, v in tie_decay_result.items()
                  if v >= 10 ** -100]
G.add_weighted_edges_from(tie_decay_edges)

edges,weights = zip(*nx.get_edge_attributes(G,'weight').items())
colors = ['r' if w > 10 ** -10 else 'b' for w in weights]
width = [1 if w > 10 ** -10 else 0.1 for w in weights]
pos = nx.spring_layout(G)

plt.figure()
nx.draw(G, pos, node_size=5, node_color='b', edgelist=edges, edge_color=colors, width=width)
plt.savefig("tie_decay_example.png")
plt.close()


# GOAL OF THIS SOFTWARE: print scraped data in Terminal
## please install PYTHON3, CHROMEDRIVER, SELENIUM, then proceed
### then enter in Terminal ```pip install -r setup.py```
#### NOW, you can run this script in Terminal with ```python3 scrape_bundles.py```
##### ♲ reuse & recycle & have a nice day ♲

# headless browser
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
# raw file data tool
import requests

# make sure you also add CHROMEDRIVER to system path (I enclosed the program in this folder so your device might find it ;)
driver = webdriver.Chrome('./chromedriver')
# read here if your device has problems w CHROMEDRIVER PATH stuff:
# https://github.com/SeleniumHQ/selenium/wiki/ChromeDriver

# surfs Incognito => scrapes better ^_^
option = webdriver.ChromeOptions()
option.add_argument("--incognito")

# launches Chrome
browser = webdriver.Chrome(executable_path='./chromedriver', options=option)

# grab text of any
 any level inside of class="mw-category-group"
# 'div[@class="mw-category-group"]//h3'
# // any h3 in a div with class 'mw-category-group'

# surfs to this URL
page_this_time = 'Farm_buildings'
url_this_time = f"https://stardewvalleywiki.com/Category:{page_this_time}"
print(f"Let's scrape this site: {url_this_time}")
browser.get(url_this_time)

###
##
# internet lag timeout (error handling)
timeout = 30
try:
	# checking for element on page, to test connection (the only
 with id='pt-createaccount' )
	WebDriverWait(browser, timeout).until(EC.visibility_of_element_located((By.XPATH, "//li[@id='pt-createaccount']")))
except TimeoutException:
	print("This page takes too long to load-- try again?")
	browser.quit()
#
##
###
# so we are scraping the URL above {url_this_time} for info...
# confirm the next bit yourself with Inspect in your browser, then modify this script as YOU need w this guide:
# https://gist.github.com/LeCoupa/8c305ec8c713aad07b14
# ENJOY CODING
#
# the URL target has info we need: this XPATH selector id's only the HTML we want
# it's always listed (are

s) inside
 elements w this ⬇ class prop, and they are all ⬇  elements
results = browser.find_elements_by_xpath("//div[@class='mw-category-group']/ul/li/a")
# now let's collect the exact prop from these HTML elements
result_s = [x.get_attribute('title') for x in results]
# and print them all to Terminal
for result_s in result_s:
    print(result_s)
print(f"This concludes the scraping of: {url_this_time}")
# ENJOY CODING
##### ♲ reuse & recycle & have a nice day ♲


# post invoices

"""
When the user clicks on Post Invoices in PMS, it activates macro
    mcrInvoice,
which in turn runs query
    qappSuumaryReportActiveJobsOnly+HoursInvoiced
This query takes 3 paramemters, in the following order:
    * text8.Value - StartDate - e.g. 01/05/2010
    * text10,Value - EndDate - e.g. 31/05/2010
    * Combo33 - Billing Period - e.g. May 2010

"""

###########################################################################

import csv
import datetime
import itertools
import pdb
import pprint
import traceback

import ordereddict
import win32com.client

import common
from common import AsAscii, AsFloat, princ
import db
import excel
import period

############################################################################

def zap_entries():
    'Remove all recorded invoices for the period PER'
    fmt = "DELETE FROM tblInvoice WHERE InvBillingPeriod='{0}'"
    #print "zap_entries"
    sql = fmt.format(period.mmmmyyyy())
    #print sql
    db.ExecuteSql(sql)
    #db.commit()

############################################################################
def get_keys():
    return ['InvDate', 'InvBillingPeriod', 'InvBIA', 'InvUBI', 'InvWIP',
        'InvAccrual', 'InvInvoice', 'Inv3rdParty', 'InvTime',
        'InvJobCode', 'InvComments', 'InvPODatabaseCosts', 'InvCapital',
        'InvStock']

def create_insertion(jcode):
        d = ordereddict.OrderedDict(map(lambda x: [x, 0.0], get_keys()))
        d['InvDate'] = datetime.date.today().strftime('%d/%m/%Y')
        d['InvBillingPeriod'] = period.mmmmyyyy()
        d['InvJobCode'] = str(jcode)
        d['InvComments'] = "PMS " + common.get_timestamp()
        return d

def setup_insertions(tasks):
    invBillingPeriod = period.mmmmyyyy()
    #recs = db.GetTasks()
    activeJobs = set([key[0] for key in tasks.keys()])
    ignoreJobs = set(['010500', '010400', '010300', '010200', '3. Sundry', '404550'])
    jobsToCreate = activeJobs - ignoreJobs
    insertions = {}
    for jcode in jobsToCreate:
        insertions[jcode] = create_insertion(jcode)
    return insertions

def process_autos(inserts, autos, d):
    for invoice in autos.values():
        jcode = invoice['JobCode']
        insert = inserts[jcode]
        invoice_time = invoice['work']
        insert['InvTime'] += invoice_time
        party3 = invoice['expenses']
        insert['Inv3rdParty'] += party3
        if d['jobs'][jcode]['WIP']:
            insert['InvUBI'] += invoice_time
            insert['InvWIP'] += party3
        else:
            insert['InvInvoice'] += invoice['net']
        inserts[jcode] = insert

def process_manuals(inserts, manuals):
    "Process manual invoices"
    for m in manuals:
        jcode = m['JobCode']
        inserts[jcode]['InvInvoice'] += m['net']

def process_tweaks(inserts, data):
    tweaks = data['InvTweaks']
    for tweak in tweaks:
        jcode = tweak['JobCode']
        for k in ['Inv3rdParty', 'InvAccrual', 'InvBIA', 'InvInvoice', 'InvTime', 'InvUBI', 'InvWIP']:
            inserts[jcode][k] += tweak[k]
        #print tweak

def process_pos(inserts):
    # Retrieve the relevant info from the database
    #p = period.g_period
    #from_date = p.first()
    #to_date = p.last()
    fmt = "SELECT Code, Cost FROM qryPO WHERE BillingPeriod='%s'"
    sql = fmt % (period.yyyymm())
    costs = {}
    for rec in db.fetch_all(sql):
        costs[str(rec[0])] = rec[1]

    # Add PO costs to database
    for jcode in costs:
        cost = costs[jcode]
        if not inserts.has_key(jcode): continue # can happen with '3. Sundry' for example
        #    inserts[jcode] = create_insertion(jcode)
        inserts[jcode]['InvPODatabaseCosts'] = cost
        #fmt = "UPDATE tblInvoice SET InvPODatabaseCosts=%.2f WHERE InvJobCode='%s' AND InvBillingPeriod='%s'"
        #sql = fmt % (cost, job_code, p.mmmmyyyy())
        #conn.Execute(sql)

def insert_data(inserts):
    #print inserts
    #conn = db.DbOpen()
    with db.Cursor() as c:
        for ins in inserts.values():
            keys = ' , '.join(get_keys())
            #print keys
            #print ins
            values = []
            for k in get_keys():
                v = ins[k]
                if isinstance(v, str):
                    v = "'{0}'".format(v)
                else:
                    v = str(v)
                values.append(v)
            values = ' , '.join(values)

            sql = "INSERT INTO tblInvoice ({0}) VALUES ({1})".format(keys, values)
            #print sql
            #conn.execute(sql)
            c.execute(sql)
            #print sql
###########################################################################

def post_main(data, accumulated_tweaks):
    zap_entries()
    inserts = setup_insertions(data['tasks'])
    #print data.keys()
    autos = data['auto_invoices']
    process_autos(inserts, autos, data)
    manuals = data['ManualInvoices']
    process_manuals(inserts, manuals)
    process_tweaks(inserts, data)
    process_pos(inserts)
    insert_data(inserts)
    #pprint.pprint(inserts)

if  __name__ == "__main__":
    data = db.load_state()
    #print data
    post_main(data, None)
    princ("Finished")


import webbrowser, sys, pyperclip
'''This a simple and straight program that displays the power of the webbrowser module. To be able to run it from the command line, need to run it as a batchfile.
The conditions however makes it possible so that at all times either one of this variables contains something either with command line arguments or clipboard.'''

if len(sys.argv) > 1:
    open_myweb= ' '.join(sys.argv[1:])
elif len(sys.argv) == 1:
    open_myweb = pyperclip.paste()

webbrowser.open(open_myweb)


import requests
import json, os, sys
from base64_extract import convert_2_base64
from pprint import pprint
from multiprocessing import Pool
import random
import time

enroll_url = 'http://localhost:5000/trueid/enroll'
predict_url = 'http://localhost:5000/trueid/predict'

raw_dir = "biometric/dataset/raw/orig"
list_person_names = ["0", "1", "2", "3", "4", "5", "6", "7", "9", "10", "14"]

import logging
logger = logging.getLogger("client_simulation")

log_file = "client_simulation.log"
logging.basicConfig(filename=log_file, format='%(asctime)s : %(levelname)s : %(message)s', filemode='w')
logging.root.setLevel(level=logging.INFO)
logger.info("running %s" % ' '.join(sys.argv))

def run_simulate_one_process(client_id):
    #client_id = os.getpid()
    enrolled_person = set()
    num_enroll_requests = random.randint(10, 20)
    num_predict_requests = random.randint(20, 40)
    logger.info("Client {} info: num_enroll_requests = {}, num_predict_requests = {}.".format(client_id, num_enroll_requests, num_predict_requests))

    i_request = 0
    i_predict = 0
    while i_request < num_enroll_requests or i_predict < num_predict_requests:
        prob = random.random()
        if len(enrolled_person) == 0 or prob < 0.5: # do the enroll
            time_2_wait = random.randint(1, 9)
            time.sleep(time_2_wait)

            person_name = random.choice(list_person_names)
            payload = {
                        "PII": {
                                "person_name": person_name,
                                },
                        "images": []
                        }
            face_dir = os.path.join(raw_dir, person_name)
            encode_output_str = convert_2_base64(face_dir)
            payload["images"] = encode_output_str

            headers = {'Content-type': 'application/json'}
            print("Client {}: Sending enroll request for person name = {}".format(client_id, person_name))
            logger.info("Client {}: Sending enroll request for person name = {}".format(client_id, person_name))
            r = requests.post(enroll_url, data=json.dumps(payload), headers = headers)
            print(r.status_code)
            res_data = json.loads(r.text)
            enrolled_person.add(person_name)
            print("Client {}: Received enroll response from server: {}".format(client_id, res_data))
            logger.info("Client {}: Received enroll response from server: {}".format(client_id, res_data))
            pprint(res_data)
        else: # do the prediction
            time_2_wait = random.randint(1, 9)
            time.sleep(time_2_wait)

            person_name = random.choice(list(enrolled_person))
            payload = {
                        "images": []
                        }
            face_dir = os.path.join(raw_dir, person_name)
            encode_output_str = convert_2_base64(face_dir)
            payload["images"] = encode_output_str

            headers = {'Content-type': 'application/json'}
            print("Client {}: Sending predict request for person name = {}".format(client_id, person_name))
            logger.info("Client {}: Sending predict request for person name = {}".format(client_id, person_name))
            r = requests.post(predict_url, data=json.dumps(payload), headers = headers)
            print(r.status_code)
            res_data = json.loads(r.text)

            print("Client {}: Received predict response from server: {}".format(client_id, res_data))
            logger.info("Client {}: Received predict response from server: {}".format(client_id, res_data))
            pprint(res_data)

def start_simulation():
    NUM_PROCESS = 2

    logger.info("Start simulation with 2 concurrent processes ...")

    pool = Pool(processes= NUM_PROCESS)
    pool.map(run_simulate_one_process, range(NUM_PROCESS))
    pool.close() # No more work
    pool.join() # Wait for completion

    logger.info("Finished.")

if __name__ == '__main__':
    start_simulation()


import pymongo
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score

def getText(tweet_collection):
    usernames,hashtags,text = [],[],[]
    alldata = tweet_collection.find()
    for tweet in alldata:

        ulower = str(tweet['username']).lower()
        htlower = str(tweet['hashtags']).lower()
        tlower = str(tweet['text']).lower()
        usernames.append(ulower)
        hashtags.append(htlower)
        text.append(tlower)

    return usernames,hashtags,text

def cluster(usernames,hashtags,texts):
    vectorizer = TfidfVectorizer(stop_words='english')
    n = 12

    username_vector = vectorizer.fit_transform(usernames)
    hashtag_vector = vectorizer.fit_transform(hashtags)
    text_vector = vectorizer.fit_transform(texts)

    usernames_cluster = KMeans(n_clusters=n,init='k-means++',max_iter=100,n_init=1)
    usernames_cluster.fit(username_vector)

    hashtags_cluster = KMeans(n_clusters=n,init='k-means++',max_iter=100,n_init=1)
    hashtags_cluster.fit(hashtag_vector)

    text_cluster = KMeans(n_clusters=n,init='k-means++',max_iter=100,n_init=1)
    text_cluster.fit(text_vector)

    return vectorizer,usernames_cluster,hashtags_cluster,text_cluster

def topResult(vectorizer,group):
    n = 12
    centroids = group.cluster_centers_.argsort()[:,::-1]
    text = vectorizer.get_feature_names()
    for centre in range(n):
        print ("CLUSTER %d:" % centre)
        for i in centroids[centre, :5]:
            print (' %s' % text[i])

if __name__ == "__main__":
    MONGO_HOST = 'mongodb://localhost/twitterdb'
    client = pymongo.MongoClient(MONGO_HOST)
    db = client.twitterdb
    tweet_collection = db.tweets

    sc,ht,t = getText(tweet_collection)
    vec,scK,htK,tK = cluster(sc,ht,t)

    print("++++ USERNAME CLUSTERS ++++")
    topResult(vec,scK)
    print("++++ HASHTAG CLUSTERS ++++")
    topResult(vec,htK)
    print("++++ TEXT CLUSTERS ++++")
    topResult(vec,tK)


# import libraries
import os
import string
import argparse
import numpy as np
import pandas as pd
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer

def read_data(folder_name):
    """
    Reads documents in folders and extract their classes, filenames and texts
    as list of lists.

    Arguments:
    'folder_name' - the folder to which the corpus are stored. It requires that
    two folders exist inside it, each with its filename as class.

    Returns:
    'label_text' - class of documents
    'filenames'  - name of files
    'texts' - list of documents
    """

    # list of text samples
    texts = []
    # dictionary mapping label name to numeric id
    labels_index = {}
    # list of label ids
    labels = []
    # list of label texts
    label_text = []
    # list of filenames
    filenames = []

    # Go through each directory in folder_name
    for name in sorted(os.listdir(folder_name)):
        path = os.path.join(folder_name, name)
        if os.path.isdir(path):
            label_id = len(labels_index)
            labels_index[name] = label_id
            for fname in sorted(os.listdir(path)):
                # reuters documents end with extension .txt
                if fname.endswith('.txt'):
                    fpath = os.path.join(path, fname)
                    f = open(fpath, encoding = 'latin-1')
                    t = f.read()
                    texts.append(t)
                    f.close()
                    labels.append(label_id)
                    label_text.append(name)
                    filenames.append(fname)

    print('Found %s texts.' % len(texts))

    return (label_text, filenames, texts)

# store stripped and lowercased list of sentences
def clean_corpus(corpus):
    """
    Cleans the corpus with basic pre-processing: lowercase,
    digit and ponctuation removal, then strips on any space.

    Arguments:
    'corpus' - list of texts acquired from read_data() function

    Returns:
    'clean_texts' - cleaned and pre-processed list of tokens.
    """

    # for the whole corpus already read
    clean_texts = []

    for text in corpus:
        # remove any punctuation or numbers
        text = text.translate(str.maketrans('','',string.punctuation))
        text = text.translate(str.maketrans('','','1234567890'))
        text = text.lower()
        clean_texts.append(text)

    return clean_texts

def get_vectors(corpus, labels, filenames, svd, top, tf_idf = False):
    """
    Gets document vectors based on the argument passed in the command line.

    Arguments:
    'corpus' - cleaned corpus, in tokens.
    'labels' - labels of each document tokens in corpus
    'filenames' - file names of each document tokens in corpus
    'svd' - number of dimensions for TruncatedSVD. Default is "None", i.e. no truncation.
    'top' - most common top words in the corpus are returned.
    'tf_idf' - If True, a tf-idf vector is returned.

    Returns:
    'df' - desired document vector based on arguments passed.
    """

    # token_pattern added to include 1 letter word as default is 2
    count_vect = CountVectorizer(token_pattern = r"(?u)\b\w+\b", max_features=top)

    # transform corpus
    X_counts = count_vect.fit_transform(corpus)

    if not tf_idf and svd != None:
        # initiate trucantedSVD
        svd = TruncatedSVD(n_components=svd)

        #transform term-document matrix with trucantedSVD
        X_svd = svd.fit_transform(X_counts)

        df = pd.DataFrame(X_svd)
        df.insert(loc=0, column='topic_class', value=np.asarray(labels))
        df.insert(loc=1, column='topic_filename', value=np.asarray(filenames))
        return df

    if tf_idf and svd == None:
        tf_transformer = TfidfTransformer(use_idf=True).fit(X_counts)
        X_tf = tf_transformer.transform(X_counts)
        df = pd.DataFrame(X_tf.toarray(), columns=count_vect.get_feature_names())
        df.insert(loc=0, column='topic_class', value=np.asarray(labels))
        df.insert(loc=1, column='topic_filename', value=np.asarray(filenames))
        return df

    if tf_idf and svd != None:
        tf_transformer = TfidfTransformer(use_idf=True).fit(X_counts)
        X_tf = tf_transformer.transform(X_counts)

        # initiate trucantedSVD
        svd = TruncatedSVD(n_components=svd)

        #transform term-document matrix with trucantedSVD
        X_svd = svd.fit_transform(X_tf)

        df = pd.DataFrame(X_svd)
        df.insert(loc=0, column='topic_class', value=np.asarray(labels))
        df.insert(loc=1, column='topic_filename', value=np.asarray(filenames))
        return df

    if not tf_idf and svd == None:
        df = pd.DataFrame(X_counts.toarray(), columns=count_vect.get_feature_names())
        df.insert(loc=0, column='topic_class', value=np.asarray(labels))
        df.insert(loc=1, column='topic_filename', value=np.asarray(filenames))
        return df

# command line interpreter
if __name__ == "__main__":

    # create ArgumentParser object
    parser = argparse.ArgumentParser(description="Generate term-document matrix.")
    parser.add_argument("-T", "--tfidf", action="store_true", help="Apply tf-idf to the matrix.")
    parser.add_argument("-S", "--svd", metavar="N", dest="svddims", type=int,
                        default=None,
                        help="Use TruncatedSVD to truncate to N dimensions")
    parser.add_argument("-B", "--base-vocab", metavar="M", dest="basedims",
                        type=int, default=None,
                        help="Use the top M dims from the raw counts before further processing")
    parser.add_argument("foldername", type=str,
                        help="The base folder name containing the two topic subfolders.")
    parser.add_argument("outputfile", type=str,
                        help="The name of the output file for the matrix data.")

    args = parser.parse_args()

    print("Loading data from directory {}.".format(args.foldername))
    labels, filenames, texts = read_data(args.foldername)

    if not args.basedims:
        print("Using full vocabulary.")
    else:
        print("Using only top {} terms by raw count.".format(args.basedims))

    if args.tfidf:
        print("Applying tf-idf to raw counts.")

    if args.svddims:
        print("Truncating matrix to {} dimensions via singular value decomposition.".format(args.svddims))

    # THERE ARE SOME ERROR CONDITIONS YOU MAY HAVE TO HANDLE WITH CONTRADICTORY
    # PARAMETERS.
    df = get_vectors(corpus=clean_corpus(texts),labels = labels, filenames = filenames, svd = args.svddims, top = args.basedims, tf_idf = args.tfidf)

    print("Writing matrix to {}.".format(args.outputfile))
    output = open(args.outputfile, 'w')
    output.write(df.to_string())
    output.close()


# -*- coding: utf-8 -*-

# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import proto  # type: ignore

from google.protobuf import timestamp_pb2 as timestamp  # type: ignore

__protobuf__ = proto.module(
    package='google.cloud.resourcesettings.v1',
    manifest={
        'SettingValue',
        'Setting',
        'Value',
        'ListSettingsRequest',
        'ListSettingsResponse',
        'SearchSettingValuesRequest',
        'SearchSettingValuesResponse',
        'GetSettingValueRequest',
        'LookupEffectiveSettingValueRequest',
        'CreateSettingValueRequest',
        'UpdateSettingValueRequest',
        'DeleteSettingValueRequest',
    },
)

class SettingValue(proto.Message):
    r"""The instantiation of a setting. Every setting value is
    parented by its corresponding setting.

    Attributes:
        name (str):
            The resource name of the setting value. Must be in one of
            the following forms:

            -  ``projects/{project_number}/settings/{setting_name}/value``
            -  ``folders/{folder_id}/settings/{setting_name}/value``
            -  ``organizations/{organization_id}/settings/{setting_name}/value``

            For example,
            "/projects/123/settings/gcp-enableMyFeature/value".
        value (google.cloud.resourcesettings_v1.types.Value):
            The value of the setting. The data type of
            [Value][google.cloud.resourcesettings.v1.Value] must always
            be consistent with the data type defined by the parent
            setting.
        etag (str):
            A fingerprint used for optimistic concurrency. See
            [UpdateSettingValue][google.cloud.resourcesettings.v1.ResourceSettingsService.UpdateSettingValue]
            for more details.
        read_only (bool):
            Output only. A flag indicating that this setting value
            cannot be modified. This flag is inherited from its parent
            setting and is for convenience purposes. See
            [Setting.read_only][google.cloud.resourcesettings.v1.Setting.read_only]
            for more details.
        update_time (google.protobuf.timestamp_pb2.Timestamp):
            Output only. The timestamp indicating when
            the setting value was last updated.
    """

    name = proto.Field(proto.STRING, number=1)

    value = proto.Field(proto.MESSAGE, number=2,
        message='Value',
    )

    etag = proto.Field(proto.STRING, number=3)

    read_only = proto.Field(proto.BOOL, number=4)

    update_time = proto.Field(proto.MESSAGE, number=5,
        message=timestamp.Timestamp,
    )

class Setting(proto.Message):
    r"""The schema for setting values. At a given Cloud resource, a
    setting can parent at most one setting value.

    Attributes:
        name (str):
            The resource name of the setting. Must be in one of the
            following forms:

            -  ``projects/{project_number}/settings/{setting_name}``
            -  ``folders/{folder_id}/settings/{setting_name}``
            -  ``organizations/{organization_id}/settings/{setting_name}``

            For example, "/projects/123/settings/gcp-enableMyFeature".
        display_name (str):
            The human readable name for this setting.
        description (str):
            A detailed description of what this setting
            does.
        read_only (bool):
            A flag indicating that values of this setting
            cannot be modified (see documentation of the
            specific setting for updates and reasons).
        data_type (google.cloud.resourcesettings_v1.types.Setting.DataType):
            The data type for this setting.
        default_value (google.cloud.resourcesettings_v1.types.Value):
            The value received by
            [LookupEffectiveSettingValue][google.cloud.resourcesettings.v1.ResourceSettingsService.LookupEffectiveSettingValue]
            if no setting value is explicitly set.

            Note: not all settings have a default value.
    """
    class DataType(proto.Enum):
        r"""The data type for setting values of this setting. See
        [Value][google.cloud.resourcesettings.v1.Value] for more details on
        the available data types.
        """
        DATA_TYPE_UNSPECIFIED = 0
        BOOLEAN = 1
        STRING = 2
        STRING_SET = 3
        ENUM_VALUE = 4

    name = proto.Field(proto.STRING, number=1)

    display_name = proto.Field(proto.STRING, number=2)

    description = proto.Field(proto.STRING, number=3)

    read_only = proto.Field(proto.BOOL, number=4)

    data_type = proto.Field(proto.ENUM, number=5,
        enum=DataType,
    )

    default_value = proto.Field(proto.MESSAGE, number=6,
        message='Value',
    )

class Value(proto.Message):
    r"""The data in a setting value.

    Attributes:
        boolean_value (bool):
            Defines this value as being a boolean value.
        string_value (str):
            Defines this value as being a string value.
        string_set_value (google.cloud.resourcesettings_v1.types.Value.StringSet):
            Defines this value as being a StringSet.
        enum_value (google.cloud.resourcesettings_v1.types.Value.EnumValue):
            Defines this value as being a Enum.
    """
    class StringSet(proto.Message):
        r"""A string set value that can hold a set of strings. The
        maximum length of each string is 60 characters and there can be
        a maximum of 50 strings in the string set.

        Attributes:
            values (Sequence[str]):
                The strings in the set
        """

        values = proto.RepeatedField(proto.STRING, number=1)

    class EnumValue(proto.Message):
        r"""A enum value that can hold any enum type setting values.
        Each enum type is represented by a number, this representation
        is stored in the definitions.

        Attributes:
            value (str):

        """

        value = proto.Field(proto.STRING, number=1)

    boolean_value = proto.Field(proto.BOOL, number=1, oneof='value')

    string_value = proto.Field(proto.STRING, number=2, oneof='value')

    string_set_value = proto.Field(proto.MESSAGE, number=3, oneof='value',
        message=StringSet,
    )

    enum_value = proto.Field(proto.MESSAGE, number=4, oneof='value',
        message=EnumValue,
    )

class ListSettingsRequest(proto.Message):
    r"""The request for ListSettings.

    Attributes:
        parent (str):
            Required. The Cloud resource that parents the setting. Must
            be in one of the following forms:

            -  ``projects/{project_number}``
            -  ``projects/{project_id}``
            -  ``folders/{folder_id}``
            -  ``organizations/{organization_id}``
        page_size (int):
            Unused. The size of the page to be returned.
        page_token (str):
            Unused. A page token used to retrieve the
            next page.
    """

    parent = proto.Field(proto.STRING, number=1)

    page_size = proto.Field(proto.INT32, number=2)

    page_token = proto.Field(proto.STRING, number=3)

class ListSettingsResponse(proto.Message):
    r"""The response from ListSettings.

    Attributes:
        settings (Sequence[google.cloud.resourcesettings_v1.types.Setting]):
            A list of settings that are available at the
            specified Cloud resource.
        next_page_token (str):
            Unused. A page token used to retrieve the
            next page.
    """

    @property
    def raw_page(self):
        return self

    settings = proto.RepeatedField(proto.MESSAGE, number=1,
        message='Setting',
    )

    next_page_token = proto.Field(proto.STRING, number=2)

class SearchSettingValuesRequest(proto.Message):
    r"""The request for SearchSettingValues.

    Attributes:
        parent (str):
            Required. The Cloud resource that parents the setting. Must
            be in one of the following forms:

            -  ``projects/{project_number}``
            -  ``projects/{project_id}``
            -  ``folders/{folder_id}``
            -  ``organizations/{organization_id}``
        page_size (int):
            Unused. The size of the page to be returned.
        page_token (str):
            Unused. A page token used to retrieve the
            next page.
    """

    parent = proto.Field(proto.STRING, number=1)

    page_size = proto.Field(proto.INT32, number=2)

    page_token = proto.Field(proto.STRING, number=3)

class SearchSettingValuesResponse(proto.Message):
    r"""The response from SearchSettingValues.

    Attributes:
        setting_values (Sequence[google.cloud.resourcesettings_v1.types.SettingValue]):
            All setting values that exist on the
            specified Cloud resource.
        next_page_token (str):
            Unused. A page token used to retrieve the
            next page.
    """

    @property
    def raw_page(self):
        return self

    setting_values = proto.RepeatedField(proto.MESSAGE, number=1,
        message='SettingValue',
    )

    next_page_token = proto.Field(proto.STRING, number=2)

class GetSettingValueRequest(proto.Message):
    r"""The request for GetSettingValue.

    Attributes:
        name (str):
            Required. The name of the setting value to get. See
            [SettingValue][google.cloud.resourcesettings.v1.SettingValue]
            for naming requirements.
    """

    name = proto.Field(proto.STRING, number=1)

class LookupEffectiveSettingValueRequest(proto.Message):
    r"""The request for LookupEffectiveSettingValue.

    Attributes:
        name (str):
            Required. The setting value for which an effective value
            will be evaluated. See
            [SettingValue][google.cloud.resourcesettings.v1.SettingValue]
            for naming requirements.
    """

    name = proto.Field(proto.STRING, number=1)

class CreateSettingValueRequest(proto.Message):
    r"""The request for CreateSettingValue.

    Attributes:
        parent (str):
            Required. The name of the setting for which a value should
            be created. See
            [Setting][google.cloud.resourcesettings.v1.Setting] for
            naming requirements.
        setting_value (google.cloud.resourcesettings_v1.types.SettingValue):
            Required. The setting value to create. See
            [SettingValue][google.cloud.resourcesettings.v1.SettingValue]
            for field requirements.
    """

    parent = proto.Field(proto.STRING, number=1)

    setting_value = proto.Field(proto.MESSAGE, number=2,
        message='SettingValue',
    )

class UpdateSettingValueRequest(proto.Message):
    r"""The request for UpdateSettingValue.

    Attributes:
        setting_value (google.cloud.resourcesettings_v1.types.SettingValue):
            Required. The setting value to update. See
            [SettingValue][google.cloud.resourcesettings.v1.SettingValue]
            for field requirements.
    """

    setting_value = proto.Field(proto.MESSAGE, number=1,
        message='SettingValue',
    )

class DeleteSettingValueRequest(proto.Message):
    r"""The request for DeleteSettingValue.

    Attributes:
        name (str):
            Required. The name of the setting value to delete. See
            [SettingValue][google.cloud.resourcesettings.v1.SettingValue]
            for naming requirements.
    """

    name = proto.Field(proto.STRING, number=1)

__all__ = tuple(sorted(__protobuf__.manifest))


from django.conf import settings
from django.db.models import F, Count
from ob.models import Listing, ExchangeRate
import json
import requests

def bootstrap(more=False):
    if more:
        from ob.phixtures.good_nodes import more_nodes as good_nodes
    else:
        from ob.phixtures.good_nodes import good_nodes

    from ob.models import Profile
    for p in good_nodes:
        p, pc = Profile.objects.get_or_create(pk=p)
        if p.should_update():
            try:
                p.sync(testnet=False)
            except requests.exceptions.ReadTimeout:
                print("read timeout")
        else:
            print('skipping profile')

def get_exchange_rates():
    rates_url = settings.OB_MAINNET_HOST + 'exchangerates/'
    response = requests.get(rates_url,
                            timeout=settings.CRAWL_TIMEOUT,
                            auth=settings.OB_API_AUTH,
                            verify=settings.OB_CERTIFICATE
                            )
    if response.status_code == 200:
        forex_data = json.loads(response.content.decode('utf-8'))
        for k, v in forex_data.items():
            updated = ExchangeRate.objects.filter(symbol__exact=k).update(rate=v)
            if updated == 0:
                fx, fx_c = ExchangeRate.objects.get_or_create(symbol=k)
                fx.rate = v
                fx.save()

def update_price_values():
    # Listing.update(stories_filed=F('stories_filed') + 1)
    qs_currency_count = Listing.objects.values('pricing_currency')\
        .annotate(cur_count=Count('pricing_currency'))\
        .filter(cur_count__gte=1).order_by('-cur_count')
    distinct_currencies = [v['pricing_currency'] for v in list(qs_currency_count)]
    for c_symbol in distinct_currencies:
        try:
            c = ExchangeRate.objects.get(symbol=c_symbol)
            a = Listing.objects.filter(pricing_currency=c_symbol).update(
                price_value=F('price') / float(c.base_unit) / float(c.rate))
        except ExchangeRate.DoesNotExist:
            print('could not update price_values of ' + c_symbol + ' denominated listings')

def update_verified():
    verified_url = 'https://search.ob1.io/verified_moderators'
    response = requests.get(verified_url)
    if response.status_code == 200:
        from ob.models import Profile
        verified_data = json.loads(response.content.decode('utf-8'))
        verified_pks = [p['peerID'] for p in verified_data['moderators']]
        Profile.objects.filter().update(verified=False)
        Profile.objects.filter(pk__in=verified_pks).update(verified=True)


# -*- coding: utf-8 -*-
"""
Created on Tue Dec 22 15:25:34 2020

@author: 0100904
不能读取加密文件

"""

import xlrd
import xlwt
from datetime import date,datetime

def read_excel():
# 打开文件
    workbook = xlrd.open_workbook(r'd:/RIC组装表2020.xlsx')
# 获取所有sheet
    sheet_name = workbook.sheet_names()[0]
    sheet = workbook.sheet_by_name(sheet_name)

#显示表内容
    for i in range(6,sheet.nrows):
            for j in range(0,sheet.ncols):
                print(sheet.cell(i,j).value)

if __name__ == '__main__':
    read_excel()


# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""service-management convert-config command."""

import os

from apitools.base.py import encoding

from googlecloudsdk.api_lib.service_management import services_util

from googlecloudsdk.calliope import base
from googlecloudsdk.calliope import exceptions
from googlecloudsdk.core import log

@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA)
class ConvertConfig(base.Command):
  """Convert Swagger specification to Google service configuration.

  DEPRECATED: This command is deprecated and will be removed soon.
  """

  @staticmethod
  def Args(parser):
    """Args is called by calliope to gather arguments for this command.

    Args:
      parser: An argparse parser that you can use to add arguments that go
          on the command line after this command. Positional arguments are
          allowed.
    """
    parser.add_argument(
        'open_api_file',
        help='The file path containing the Open API specification to convert.')
    parser.add_argument(
        'output_file', nargs='?', default='',
        help=('The file path of the output file containing the converted '
              'configuration. Output to standard output if omitted.'))

  def Run(self, args):
    """Run 'service-management convert-config'.

    Args:
      args: argparse.Namespace, The arguments that this command was invoked
          with.

    Returns:
      The response from the ConvertConfig API call.

    Raises:
      IOError: An IOError is returned if the input file cannot be read, or
          the output file cannot be written to.
    """
    log.warn('This command is deprecated and will be removed soon.')

    messages = services_util.GetMessagesModule()
    client = services_util.GetClientInstance()

    # TODO(user): Add support for swagger file references later
    # This requires the API to support multiple files first. b/23353397
    try:
      with open(args.open_api_file) as f:
        open_api_spec = messages.OpenApiSpec(openApiFiles=[
            messages.ConfigFile(
                filePath=os.path.basename(args.open_api_file),
                contents=f.read())
        ])
    except IOError:
      raise exceptions.ToolException.FromCurrent(
          'Cannot open {f} file'.format(f=args.open_api_file))

    request = messages.ConvertConfigRequest(openApiSpec=open_api_spec)

    converted_config = client.v1.ConvertConfig(request)
    diagnostics = converted_config.diagnostics
    if diagnostics:
      kind = messages.Diagnostic.KindValueValuesEnum
      for diagnostic in diagnostics:
        logger = log.error if diagnostic.kind == kind.ERROR else log.warning
        logger('{l}: {m}'.format(l=diagnostic.location, m=diagnostic.message))

    service = converted_config.serviceConfig
    if service:
      if args.output_file:
        try:
          with open(args.output_file, 'w') as out:
            out.write(encoding.MessageToJson(service))
        except IOError:
          raise exceptions.ToolException.FromCurrent(
              'Cannot open output file \'{f}\''.format(f=args.output_file))
      else:
        return service

  def Format(self, unused_args):
    return 'json'