#!/usr/bin/env python
import random
import os
import pyttsx3
import speech_recognition as sr
import datetime
import shutil
import glob
import pandas as pd
from pyaudio import PyAudio
from tkinter.filedialog import askdirectory
from tkinter import Tk
import os
import hashlib
from pathlib import Path
import datetime
import glob
import os
import pyttsx3
import speech_recognition as sr
import datetime
engine = pyttsx3.init('sapi5')
voices = engine.getProperty('voices')
engine.setProperty('voice', voices[1].id)
def speak(audio: object) -> object:
    engine.say(audio)
    engine.runAndWait()
def takeCommand():
    r = sr.Recognizer()

    with sr.Microphone() as source:

        print("Listening...")
        r.pause_threshold = 1
        audio = r.listen(source)

    try:
        print("Recognizing...")
        query = r.recognize_google(audio, language='en-in')
        print(f"User said: {query}\n")

    except Exception as e:
        print(e)
        print("Unable to Recognize your voice.")
        return "None"

    return query
def wishMe():
    hour = int(datetime.datetime.now().hour)
    if hour >= 0 and hour < 12:
        speak("Good Morning  !")

    elif hour >= 12 and hour < 18:
        speak("Good Afternoon !")

    else:
        speak("Good Evening  !")
def display_instructions():
    pass
def introduction_and_instructions():
    speak("Hello....!")
    wishMe()
    speak("Welcome to automated file management system")
    speak("This project deals with  memory allocation and deallocation as well as basic file operations using voice commands which was given by user")
    speak("As this is a voice assistant........")
    speak("please give the commands when you see listening.... in the screen")
    speak("it is my primary responsibility to display the user manual")
    '''display_instructions()'''
    speak("make sure you give proper commands to me .......so that i can compute your requests")
    speak("now...Handing over to you")
'''class Automatic_File_Management:
    def init(self):
        pass
    def get_folder_statistics(self, folder):
        d = {}
        creation = []
        last_modification = []
        size = []
        file_name = []
        l = os.listdir(format(folder))
        for ele in l:
            file_name.append(ele)
            c = datetime.datetime.fromtimestamp(os.path.getctime(f'{folder}/{ele}'))
            m = datetime.datetime.fromtimestamp(os.path.getmtime(f'{folder}/{ele}'))
            creation.append(str(c).split()[0])
            last_modification.append(str(m).split()[0])
            size.append((os.path.getsize(f'{folder}/{ele}') / (1024 * 1024)))
        d["File"] = file_name
        d['Created'] = creation
        d['last_modified'] = last_modification
        d['Size(MB)'] = size
        df = pd.DataFrame(d)
        return df
    def get_files_with_maximum_storage(self, df: object) -> object:
        return df.sort_values('Size(MB)', ascending=False)
    def get_files_within_time(self, df, time):
        year = time.split('-')[0]
        month = time.split('-')[1]
        day = time.split('-')[2]
        sorted_statistics = df.sort_values('last_modified')
        sorted_statistic_upto_date = df[df['last_modified'] < year]
        maximum = self.get_files_with_maximum_storage(sorted_statistic_upto_date)
        return maximum
    def get_actual_file_location(self, file_name):
        file_locations = []
        desktop_path = 'C:/Users/91938/Desktop/**/{}'.format(file_name)
        documen_path = 'C:/Users/91938/Documents/**/{}'.format(file_name)
        downloads_path = 'C:/Users/91938/Downloads/**/{}'.format(file_name)
        music_path = 'C:/Users/91938/Music/**/{}'.format(file_name)
        pictures_path = 'C:/Users/91938/Pictures/**/{}'.format(file_name)
        favorites_path = 'C:/Users/91938/Favorites/**/{}'.format(file_name)
        video_path = 'C:/Users/91938/Videos/**/{}'.format(file_name)
        Desktop_files = glob.glob(desktop_path, recursive=True)
        Document_files = glob.glob(documen_path, recursive=True)
        Downloads_files = glob.glob(downloads_path, recursive=True)
        Favorites_files = glob.glob(favorites_path, recursive=True)
        Music_files = glob.glob(music_path, recursive=True)
        Pictures_files = glob.glob(pictures_path, recursive=True)
        video_files = glob.glob(video_path, recursive=True)
        overall_files = [Desktop_files, Document_files, Downloads_files, Favorites_files, Music_files, Pictures_files,
                         video_files]
        for ele in overall_files:
            if len(ele) > 1:
                for ele1 in ele:
                    file_locations.append(ele1)
        return file_locations
    def deallocate(self, file_name):
        directories = ["Desktop", "Downloads", "Documents", "Videos", "Pictures", "Favorites", "Music"]
        for dir in directories:
            if file_name.find(dir) == 1:
                d = dir
        os.remove(file_name)
        print("Succesfully Deleted")
    def get_disk_storage(self):
        disk_size = shutil.disk_usage('C:/')
        disk_left_size = disk_size.used
        disk_total = disk_size.total
        return disk_left_size,disk_total
    def is_memory_allocation_possible(self,file):
        disk_left_size,disk_total=self.get_disk_storage()
        file_size = os.path.getsize(file) // ( 2 ** 30)
        if file_size < disk_total:
            return True
        else:
            return False'''
class Automatic_File_Management:
    def init(self):
        pass
    def listdirs(self, root):
        l = []
        for path, subdirs, files in os.walk(root):
            for name in files:
                l.append(os.path.join(path, name))
        return l
    def get_folder_statistics(self, folder):
        d = {}
        creation = []
        last_modification = []
        size = []
        file_name = []
        l = os.listdir(folder)
        for ele in l:
            if os.path.isdir(os.path.join(folder,ele)):
                ld = self.listdirs(os.path.join(folder,ele))
                for i in ld:
                    file_name.append(i)
                    c = datetime.datetime.fromtimestamp(os.path.getctime(i))
                    m = datetime.datetime.fromtimestamp(os.path.getmtime(i))
                    creation.append(str(c).split()[0])
                    last_modification.append(str(m).split()[0])
                    size.append((os.path.getsize(i) / (1024 * 1024)))
            file_name.append(ele)
            c = datetime.datetime.fromtimestamp(os.path.getctime(f'{folder}/{ele}'))
            m = datetime.datetime.fromtimestamp(os.path.getmtime(f'{folder}/{ele}'))
            creation.append(str(c).split()[0])
            last_modification.append(str(m).split()[0])
            size.append((os.path.getsize(f'{folder}/{ele}') / (1024 * 1024)))
        d["File"] = file_name
        d['Created'] = creation
        d['last_modified'] = last_modification
        d['Size(MB)'] = size
        df = pd.DataFrame(d)
        return df
    def get_files_with_maximum_storage(self, df: object) -> object:
        return df.sort_values('Size(MB)', ascending=False)
    def get_files_within_time(self, df, time):
        year = time.split('-')[0]
        month = time.split('-')[1]
        day = time.split('-')[2]
        sorted_statistics = df.sort_values('last_modified')
        sorted_statistic_upto_date = df[df['last_modified'] < year]
        maximum = self.get_files_with_maximum_storage(sorted_statistic_upto_date)
        return maximum
    def get_actual_file_location(self, file_name):
        file_locations = []
        desktop_path = 'C:/Users/91938/Desktop/**/{}'.format(file_name)
        documen_path = 'C:/Users/91938/Documents/**/{}'.format(file_name)
        downloads_path = 'C:/Users/91938/Downloads/**/{}'.format(file_name)
        music_path = 'C:/Users/91938/Music/**/{}'.format(file_name)
        pictures_path = 'C:/Users/91938/Pictures/**/{}'.format(file_name)
        favorites_path = 'C:/Users/91938/Favorites/**/{}'.format(file_name)
        video_path = 'C:/Users/91938/Videos/**/{}'.format(file_name)
        Desktop_files = glob.glob(desktop_path, recursive=True)
        Document_files = glob.glob(documen_path, recursive=True)
        Downloads_files = glob.glob(downloads_path, recursive=True)
        Favorites_files = glob.glob(favorites_path, recursive=True)
        Music_files = glob.glob(music_path, recursive=True)
        Pictures_files = glob.glob(pictures_path, recursive=True)
        video_files = glob.glob(video_path, recursive=True)
        overall_files = [Desktop_files, Document_files, Downloads_files, Favorites_files, Music_files, Pictures_files,
                         video_files]
        for ele in overall_files:
            if len(ele) > 1:
                for ele1 in ele:
                    file_locations.append(ele1)
        return file_locations
    def deallocate(self, file_name):
        directories = ["Desktop", "Downloads", "Documents", "Videos", "Pictures", "Favorites", "Music"]
        for dir in directories:
            if file_name.find(dir) == 1:
                d = dir
        os.remove(file_name)
        print("Succesfully Deleted")
    def get_disk_storage(self):
        disk_size = shutil.disk_usage('C:/')
        disk_left_size = disk_size.used
        disk_total = disk_size.total
        return disk_left_size,disk_total
    def is_memory_allocation_possible(self,file):
        disk_left_size, disk_total = self.get_disk_storage()
        file_size = os.path.getsize(file) // ( 2 ** 30)
        if file_size < disk_total:
            return True
        else:
            return False
    def check_file_with_extenstion_if_it_is_present(self, file):
        df = pd.read_csv("C:/Users/91938/Desktop/Automated OS/allfiles.csv")
        l = list(df["File"])
        txt = []
        ppt = []
        pdf = []
        docx = []
        java = []
        python = []
        excel_and_csv = []
        ipy = []
        jpg_or_jpeg = []
        png = []
        c = []
        for ele in l:
            if ele.endswith(".txt"):
                txt.append(ele)
            elif ele.endswith(".c"):
                c.append(ele)
            elif ele.endswith(".png"):
                png.append(ele)
            elif ele.endswith(".jpg") or ele.endswith(".jpeg"):
                jpg_or_jpeg.append(ele)
            elif ele.endswith(".java"):
                java.append(ele)
            elif ele.endswith(".py"):
                python.append(ele)
            elif ele.endswith(".ipynb"):
                ipy.append(ele)
            elif ele.endswith(".xlsx") or ele.endswith(".csv"):
                excel_and_csv.append(ele)
            elif ele.endswith(".docx"):
                docx.append(ele)
            elif ele.endswith(".pdf"):
                pdf.append(ele)
            elif ele.endswith(".pptx"):
                ppt.append(ele)
        if file.endswith(".txt"):
            r = []
            for ele in txt:
                if ele.endswith(file):
                    r.append(1)
            return len(r)

        elif file.endswith(".pdf"):
            r = []
            for ele in pdf:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".pptx"):
            r = []
            for ele in ppt:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".docx"):
            r = []
            for ele in docx:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".java"):
            r = []
            for ele in java:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".py"):
            r = []
            for ele in python:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".ipynb"):
            r = []
            for ele in ipy:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".xlsx") or file.endswith(".csv"):
            r = []
            for ele in excel_and_csv:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".jpeg") or file.endswith(".jpg"):
            r = []
            for ele in jpg_or_jpeg:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".png"):
            r = []
            for ele in png:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".c"):
            r = []
            for ele in c:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
class txtfiles:
    def _init_(self):
        pass

    def write_file(self, txt, file_name=None):
        if file_name != None:
            f = open(file_name, "w")
            f.writelines(txt)
            f.close()
        else:
            file_name = "C:/Users/{}/Automated OS/Files/Text/{}.txt".format(os.getlogin(),
                                                                            random.choice([i for i in range(1000)]))
            f = open(file_name, "w")
            f.writelines(txt)
            f.close()

    def read_file(self, file_name):
        file_name
        f = open(file_name, "r")
        l = []
        for i in f.readlines():
            l.append(i)
        f.close()
        return l

    def create_file(self, txt, file_name=None):

         if file_name != None:
            file_name1 = "C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(file_name)
            f = open(file_name1, "w")
            f.writelines(txt)
            f.close()

    def update_file(self, txt, file_name):
        if file_name != None:
            file_name1  = "C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(file_name)
            f = open(file_name1, "a")
            f.writelines(txt)
            f.close()
        else:
            speak("new file being created ")
            self.create_file(self, txt, file_name)

    def delete_file(self, file_name):
        if file_name != None:
            os.remove("C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(file_name))
        else:
            speak("no file exists with name {} to delete".format(file_name))

    def copy(self, file_name, dest=None):

        if dest != None:
            shutil.copy(file_name, dest)
        else:
            path = 'C:/Users/91938/Desktop/Automated OS/Files/Text/{}-{}.txt'.format(file_name,"Copy")
            shutil.copy(file_name,'C:/Users/91938/Desktop/Automated OS/Files/Text')

    def move(self, file, dest):
        shutil.move(file, dest)
class Cluster_the_Files:
    def _init_(self):
        pass

    def get_extention(self, file_name):
        return str(file_name).split('.')[-1]

    def create_copy_of_that_file(self, filename):
        path = str(filename).split('.')[-2]
        copied_file = path + '-Copy'
        copy = copied_file + "." + self.get_extention(filename)
        shutil.copyfile(filename, copy)
        return copy

    def cluster_files_from_dir(self, dir):
        path = "C:/Users/91938/" + dir + "/"
        textfiles = glob.glob(path + "*.txt", recursive=True)
        docxfiles = glob.glob(path + "*.docx", recursive=True)
        pdffiles = glob.glob(path + "*.pdf", recursive=True)
        csvfiles = glob.glob(path + "*.csv", recursive=True)
        excelfiles = glob.glob(path + "*.xlsx", recursive=True)
        pptfiles = glob.glob(path + "*.ppt", recursive=True)
        javafiles = glob.glob(path + "*.java", recursive=True)
        pythonfiles = glob.glob(path + "*.py", recursive=True)
        pngfiles = glob.glob(path + "*.png", recursive=True)
        jpgfiles = glob.glob(path + "*.jpg", recursive=True)
        ipynbfiles = glob.glob(path + "*.ipynb", recursive=True)
        cfiles = glob.glob(path + "*.c", recursive=True)
        for ele in textfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Text")
        for ele in docxfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\docx")
        for ele in pptfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\PPTS")
        for ele in pngfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\PNG")
        for ele in javafiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Java Scripts")
        for ele in pythonfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Python Scripts")
        for ele in excelfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Excels and CSV's")
        for ele in csvfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Excels and CSV's")
        for ele in jpgfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\jpeg or jpg")
        for ele in ipynbfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\IPY NoteBooks")
        for ele in pdffiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\PDFS")
        print("all files are clustered from {}".format(dir))

    def cluster_files(self):
        self.cluster_files_from_dir("3D Objects")
        self.cluster_files_from_dir("Contacts")
        self.cluster_files_from_dir("Cookies")
        self.cluster_files_from_dir("Desktop")
        self.cluster_files_from_dir("Documents")
        self.cluster_files_from_dir("Downloads")
        self.cluster_files_from_dir("Favorites")
        self.cluster_files_from_dir("Links")
        self.cluster_files_from_dir("Music")
        self.cluster_files_from_dir("My Documents")
        self.cluster_files_from_dir("Pictures")
        self.cluster_files_from_dir("PycharmProjects")
        self.cluster_files_from_dir("Saved Games")
        self.cluster_files_from_dir("Recent")
        self.cluster_files_from_dir("Videos")
class Delete_duplicate_content:
    def _init_(self):
        pass

    def get_subdirectories(self, path):
        return os.listdir(path)

    def delete_duplicate_files_from_folder(self,path):

            file_list = os.listdir(path)

            unique = dict()

            for file in file_list:
                file_name = Path(os.path.join(path, file))

                if file_name.is_file():
                    fileHash = hashlib.md5(open(file_name, 'rb').read()).hexdigest()

                    if fileHash not in unique:
                        unique[fileHash] = file_name

                    else:
                        os.remove(file_name)
                        print("successfully  deleted at {}".format(path))

                else:
                    print("No Duplicates in {}".format(path))
    def delete_duplicate_files(self):
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/C Scripts")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/docx")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/Excels and CSV's")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/IPY NoteBooks")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/Java Scripts")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/jpeg or jpg")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/PDFS")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/PNG")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/PPTS")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/Python Scripts")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/Text")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Documents")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Downloads")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Pictures")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Videos")

obj1 = txtfiles()
obj2 = Automatic_File_Management()
obj3 = Delete_duplicate_content()
obj4 = Cluster_the_Files()

speak("Hello....!")
wishMe()
speak("Welcome to automated file management system")
speak("This project deals with  memory allocation and deallocation as well as basic file operations using voice commands which was given by user")
display_instructions()
speak("make sure you give proper commands to me .......so that i can compute your requests")
speak("Clustering all the files from DISK")
obj4.cluster_files()
speak("Clustering is Completed")
speak("Deleting duplicate files")
obj3.delete_duplicate_files()
speak("Duplicate files are deleted")
speak("As this is a voice assistant........")
speak("please give the commands when you see listening.... in the screen")
speak("it is my primary responsibility to display the user manual")
speak("now...Handing over to you")
obj1 = txtfiles()
obj2 = Automatic_File_Management()
obj3 = Delete_duplicate_content()
obj4 = Cluster_the_Files()
while True:
    query = takeCommand().lower()
    if "hello" in query:
        speak("hello user")
    elif 'exit' in query:
        speak("ok")
        speak("thank you")
        exit()
    elif "how are you" in query:
        speak("iam fine")
    elif "delete" in query:
        speak("please tell me the name of the file you want me to delete")
        q6 = takeCommand().lower()
        obj1.delete_file(q6)
        speak("deleted the file" + q6)
    elif "copy" in query:
        speak("tell me the name of the file you want me to copy")
        q7 = takeCommand().lower()
        speak("do you want to specify the directory to which the file needs to be copied ?")
        q8 = takeCommand()
        if "yes" in q8:
            speak("what is your destination file path with out extention?")
            q9 = takeCommand()
            q = q9[0].upper()+q9[1:]
            obj1.copy(q7, q)
            obj3.delete_duplicate_files()
        elif "no" in q8:
            obj1.copy(q7)
            speak("Alert an duplicate object is been created ")
            obj3.delete_duplicate_files()
    elif "update" in query:
        speak("mention the name of the file which you want to update without extention")
        q10 = takeCommand()
        speak("tell me the text which needs to be added into the file")
        q11 = takeCommand()
        obj1.update_file(q11, q10)
        obj3.delete_duplicate_files()
    elif "move" in query:
        speak("please mention the filename which needs to be moved")
        q12 = takeCommand().lower()
        print(q12)
        speak("mention the directory to which the file needs to be moved")
        q13 = takeCommand().lower()

        dest1  = q13[0].upper()+q13[1:]
        print(dest1,type(dest1))
        dest = "C:/Users/91938/{}".format(dest1)
        src = "C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(q12)
        obj1.move(src, dest)
    elif "open" in query:
        speak("please tell  me the name of file which you want to view or read contents")
        q4 = takeCommand().lower()
        speak("do you want me to share the content on screen ? please say yes or no")
        q5 = takeCommand().lower()
        file = "C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(q4)
        if "yes" or "yeah" in q5:
            content = obj1.read_file(file)
            for ele in content:
                print(ele)
                speak(ele)
        elif "no" in q5:
            content = obj1.read_file(file)
            for ele in content:
                speak(ele)
    elif "create" in query:
        speak("do you want me to create a text file or not ?")
        speak("please answer yes or no")
        q1 = takeCommand().lower()
        if "yes" in q1:
            speak("please mention the name of the file you want to save it as without extention")
            q2 = takeCommand().lower()
            speak("enter the content you wanted to speak")
            q3 = takeCommand().lower()
            obj1.create_file(q3, q2)
            speak("Checking if the available storage")
            if obj2.is_memory_allocation_possible("C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(q2)):
                speak("file is allocated")
                speak("file saved")
                obj3.delete_duplicate_files()
            else:
                speak("deallocation is required....")
                mdf = obj2.get_files_within_time("2021-08-01")
                speak("all files are displayed from different directories with maximum storages of files in it")
                for d in ["Desktop", "Downloads", "Documents", "Videos", "Pictures", "Favorites", "Music"]:
                    print("--------------------------------------------------------------------------------------------------------------------")
                    print("                                                 {}                                                                 ".format(dir))
                    print("--------------------------------------------------------------------------------------------------------------------")
                    print(obj2.get_files_with_maximum_storage(d))
                    print("--------------------------------------------------------------------------------------------------------------------")
                    print("--------------------------------------------------------------------------------------------------------------------")
                speak("pick up the directory from the above")
                dir2 = takeCommand().lower()
                dir2 = dir2[0].upper()+dir2[1:]
                speak("enter the index of the file from the choosen directory")
                index = takeCommand().lower()
                df = obj2.get_files_with_maximum_storage(dir2)
                file_location = obj2.get_actual_file_location(str(df["File"][index]))
                if len(file_location) > 1:
                    speak("the file you wanted to dealloacte is located at multiple locations....")
                    for ele in file_location:
                        obj2.deallocate(ele)
                    speak("deallocation is done")
                else:
                    obj2.deallocate(file_location[0])
                    obj3.delete_duplicate_files()
                    speak("deallocation is done")
        else:
            speak("sorry....im designed in a way to save the text files")
    elif "folder statistics" in query:
        speak("please enter the name of the folder or directory  from the disk")
        q14 = takeCommand().lower()
        dir  = q14[0].upper()+q14[1:]
        dir1 = "C:/Users/91938/{}".format(dir)
        statistics = obj2.get_folder_statistics(dir1)
        speak("The folder statistics will be displayed on screem")
        print("-------------------------------------------------------------------------------------------------------------------")
        print("                                                {}                                                                    ".format(dir))
        print("-------------------------------------------------------------------------------------------------------------------")
        print(statistics)
        print("-------------------------------------------------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------------------------------------------------")
    elif "file location" in query:
        speak("please tell enter the name  of file with extention as the extention is important")
        q15 = input("Enter the name of the file with extention : ")
        locations = obj2.get_actual_file_location(q15)
        speak("the location of the files are being displayed in screen")
        print(locations)
    elif "disk statistics" in query:
        disk_left, disk_total = obj2.get_disk_storage()
        speak("Your disk statistics are ")
        speak("you have used {} out of {} GB in C Drive".format(((disk_total - disk_left) // (2 * 30)),(disk_total // (2 * 30))))

    elif 'duplicate' in query:
        speak("enter the name of file with extention you wanted to duplicate")
        q16 = input("enter the name of the file you wanted to create a duplicate with extention : ")
        locations1 = obj2.get_actual_file_location(q16)
        if len(locations1) > 1:
            speak("the file you wanted to replicate is located at multiple locations....")
            speak("do you still want to make duplicate of that which occupies more storage?please answer yes or no")
            q17 = takeCommand().lower()
            if q17 == "yes":
                for ele in locations1:
                    obj4.create_copy_of_that_file(ele)
                speak("duplicates are created")
            elif q17 == "no":
                speak("pick up the index which denotes a single location of the given file")
                for i in range(len(locations1)):
                    print("{} - > {}".format(i+1,locations1[i]))
                q18 = takeCommand().lower()
                obj4.create_copy_of_that_file(locations1[int(q18)-1])
                speak("duplicate is created")
        else:
            obj4.create_copy_of_that_file(locations1[0])
            speak("duplicate is created")
        obj3.delete_duplicate_files()


import json

from django.http import HttpResponse
from django.contrib.auth.models import User, AnonymousUser
from .models import Campaign

def create_campaign(request):
    if request.method != 'POST':
        return HttpResponse(status=405)
    if isinstance(request.user, AnonymousUser):
        return HttpResponse(status=403)
    if 'players' in request.POST:
        players = (User.objects.get(id=x) for x in request.POST['players'])
    else:
        players = tuple()
    if 'name' in request.POST:
        name = request.POST['name']
    else:
        name = ''
    new_campaign = Campaign(dm=request.user, name=name)
    new_campaign.save()  # Must have an id before many to many relationships can be saved
    for player in players:
        new_campaign.players.add(player)
    new_campaign.save()
    return HttpResponse(bytes(str(new_campaign.id), 'utf8'), status=201)

def _get_user(user_id):
    query = User.objects.filter(id=user_id)
    if len(query.all()) == 0:
        return None
    return query.all()[0]

def _campaign_patch(campaign, data):
    if 'dm' in data:
        new_dm = _get_user(data['dm'])
        if new_dm is None:
            return HttpResponse(status=400)
        campaign.dm = new_dm
    if 'remove_player' in data:
        player = _get_user(data['remove_player'])
        if player is None or player not in campaign.players.all():
            return HttpResponse(status=400)
        campaign.players.remove(player)
    if 'add_player' in data:
        player = _get_user(data['add_player'])
        if player is None:
            return HttpResponse(status=400)
        campaign.players.add(player)
    if 'private' in data:
        if not isinstance(data['private'], bool):
            return HttpResponse(status=400)
        campaign.private = data['private']
    if 'name' in data:
        if not isinstance(data['private'], str):
            return HttpResponse(status=400)
        campaign.private = data['name']
    campaign.save()
    return HttpResponse(status=204)

def _campaign_get(user, campaign):
    if campaign.private and user != campaign.dm and user not in campaign.players.all():
        return HttpResponse(status=403)
    return HttpResponse(json.dumps({
        'name': campaign.name,
        'dm': campaign.dm.username,
        'players': [p.username for p in campaign.players.all()]
    }), status=200)

def campaign(request, campaign_id):
    query = Campaign.objects.filter(id=campaign_id)
    if not query.exists():
        return HttpResponse(status=404)
    campaign, = query.all()
    if request.method == 'GET':
        return _campaign_get(request.user, campaign)
    elif request.method == 'DELETE':
        if request.user != campaign.dm:
            return HttpResponse(status=403)
        campaign.delete()
        return HttpResponse(status=204)
    elif request.method == 'PATCH':
        if request.user != campaign.dm:
            return HttpResponse(status=403)
        data = json.loads(request.body, encoding='utf8')
        return _campaign_patch(campaign, data)
    return HttpResponse(status=405)


#!/usr/bin/env python3
"""Utils unit tests"""

from parameterized import parameterized
import unittest
from unittest.mock import patch
import utils

class TestAccessNestedMap(unittest.TestCase):
    """Class testing access_nested_map"""

    @parameterized.expand([
        ({"a": 1}, ("a",), 1),
        ({"a": {"b": 2}}, ("a",), {"b": 2}),
        ({"a": {"b": 2}}, ("a", "b"), 2)
    ])
    def test_access_nested_map(self, nested_map, path, expected):
        """Test access_nested_map"""
        self.assertEqual(utils.access_nested_map(nested_map, path), expected)

    @parameterized.expand([
        ({}, ("a",)),
        ({"a": 1}, ("a", "b"))
    ])
    def test_access_nested_map_exception(self, nested_map, path):
        """Test access_nested_map's raise when nested_map is not a Mapping"""
        self.assertRaises(KeyError, utils.access_nested_map, nested_map, path)

class TestGetJson(unittest.TestCase):
    """Class testing get_json"""

    @parameterized.expand([
        ("http://example.com", {"payload": True}),
        ("http://holberton.io", {"payload": False})
    ])
    def test_get_json(self, test_url, test_payload):
        """Test get_json"""
        with patch('requests.get') as mocked:
            mocked.return_value.json.return_value = test_payload
            self.assertEqual(utils.get_json(test_url), test_payload)
            mocked.assert_called_once()

class TestMemoize(unittest.TestCase):
    """ Class testing memoize """

    def test_memoize(self):
        """Test memoize"""
        class TestClass:
            def a_method(self):
                return 42

            @utils.memoize
            def a_property(self):
                return self.a_method()

        with patch.object(TestClass, 'a_method', return_value=42) as mocked:
            test_class = TestClass()
            self.assertEqual(test_class.a_property, mocked.return_value)
            self.assertEqual(test_class.a_property, mocked.return_value)
            mocked.assert_called_once()


import matplotlib.image as mpimg
import cv2
import imageio
from pathlib import Path
import os
import numpy as np
from tqdm import tqdm

import dlib
detector = dlib.get_frontal_face_detector() #切割器
def clip_image(image, boundary):
    top = np.clip(boundary.top(), 0, np.Inf).astype(np.int16)
    bottom = np.clip(boundary.bottom(), 0, np.Inf).astype(np.int16)
    left = np.clip(boundary.left(), 0, np.Inf).astype(np.int16)
    right = np.clip(boundary.right(), 0, np.Inf).astype(np.int16)
    image = cv2.resize(image[top:bottom, left:right],(128,128))
    return image

path = "E:/VGGFACE2/msra/"
path = Path(path)
file_dirs = [x for x in path.iterdir() if x.is_dir()]
print("总文件夹数目为：",len(file_dirs))    #86876
start = 22000
end = start + 2000
print("现在计算的是",start,"-",end)
counter = start
for file_dir in tqdm(file_dirs[start:end]):
    new_path = "C:/facepairs/msra_" + str(counter)
    if os.path.exists(new_path):
        counter += 1
    else:
        os.makedirs(new_path)
        counter += 1

        image_path_list = list(file_dir.glob('*.jpg'))

        image_counter = 0
        for image_path in image_path_list:
            image = np.array(mpimg.imread(image_path))
            boundarys = detector(image, 2)

            if len(boundarys) == 1:
                image_new = clip_image(image, boundarys[0])
                image_path_new = new_path + "/msra_" + str(counter) + "_" + str(image_counter) + ".jpg"  # 这里可以对保存的地点调整路径
                imageio.imsave(image_path_new, image_new)
                image_counter += 1

# with tf.device("/CPU:0"):
#     image_path = "E:/face_DATA/lfw_160_none/Aaron_Eckhart/Aaron_Eckhart_0001.jpg"
#     img = mpimg.imread(image_path)
#     #img_1 = cv2.resize(img_1, (128, 128))
#     image_path_new = "C:/face_DATA/lfw_160_none/Aaron_Eckhart/Aaron_Eckhart_0001.jpg"
#     imageio.imsave(image_path_new, img)

    #image = tf.image.random_flip_left_right(image)
    #image = tf.image.random_jpeg_quality(image,80,100)


from django.contrib import admin
from django.core.exceptions import ObjectDoesNotExist

from adminsortable2.admin import SortableAdminMixin
from django.utils.safestring import mark_safe

from .models import *

# Register your models here.

@admin.register(Slider)
class SliderAdmin(SortableAdminMixin, admin.ModelAdmin):
    list_display = ['id', 'title', 'is_active', 'order_index']
    list_filter = ['is_active']
    search_fields = ['title', 'description', 'link']

@admin.register(AdminEmails)
class AdminEmailAdmin(admin.ModelAdmin):
    list_display = ['email', 'is_active']
    search_fields = ['email']
    list_filter = ['is_active']

class SocialProfileLinksStackedInline(admin.StackedInline):
    model = SocialProfileLinks
    extra = 1

class FooterMenuLinksStackedInline(admin.StackedInline):
    model = FooterMenuLinks
    extra = 1

@admin.register(SiteSettings)
class SiteSettingsAdmin(admin.ModelAdmin):

    fieldsets = (
        (None, {'fields': (
            'footer_description',
            'header_phone',
            'header_email',
        )}),

        ('Страница: Главная', {'fields': (
            # 'index_page_text',

            'why_choose_us',
            'index_page_h1_text',
            # 'schedule',
            # 'schedule_thumbnail',
            # 'schedule_list_items',
            #
            # 'grand_service',
            # 'grand_service_thumbnail',
            # 'grand_service_list_items',
            #
            # 'grand_smeta',
            # 'grand_smeta_thumbnail',
            # 'grand_smeta_list_items',

            'icon_heading',
            'icon_1c',
            'icon_online_cashbox',
            'icon_schedule',
            'icon_grand_smeta',
            'icon_electronic_sign',
            'icon_certificates',
        )}),

        ('Страница: Гранд Смета', {'fields': (
            'grand_smeta_text',
            'grand_smeta_header_image'
        )}),

        ('Страница: Новости', {'fields': (
            'news_header_image',
        )}),

        ('Страница: Учебный центр', {'fields': (
            'schedule_header_image',
        )}),

        ('Страница: 1С', {'fields': (
            'one_c_h1_text',
            'one_c_header_image',
            'one_c_text',
        )}),

        ('Страница: Онлайн кассы', {'fields': (
            'online_cash_box_h1_text',
            'online_cash_box_header_image',
            'online_cash_box_text',
        )}),

        ('Страница: Элек. подписи', {'fields': (
            'electronic_signature_header_image',
        )}),

        ('Страница: Преподователи', {'fields': (
            'educators_h1_text',
            'educators_text',
        )})
    )

    inlines = (SocialProfileLinksStackedInline, FooterMenuLinksStackedInline)

    def has_add_permission(self, request):
        return self.model.objects.count() < 1

@admin.register(GrandSmeta)
class GrandSmeta(admin.ModelAdmin):
    list_display = ['title', 'price']
    search_fields = ['title', 'description', 'full_description']

    prepopulated_fields = {'slug': ('title',)}

    fieldsets = (
        ('Общее', {'fields': (
            'is_active',
            'thumbnail',
            'art_alt',
            'title',
            'slug',
            'short_description',
            'description',
            'price',
        )}),
        ('SEO', {'fields': (
            'page_title',
            'page_meta_keywords',
            'page_meta_description',
            'page_meta_og_title',
            'page_meta_og_description',
            'page_meta_og_image',
        )})
    )

@admin.register(Contacts)
class ContactsAdmin(admin.ModelAdmin):

    def has_add_permission(self, request):
        return self.model.objects.count() < 1

@admin.register(Feedback)
class FeedbackAdmin(admin.ModelAdmin):
    list_display = ['name', 'subject', 'is_read', 'is_reacted', 'created_at']
    list_filter = ['is_read', 'is_reacted', 'created_at']

    readonly_fields = ['is_read', 'created_at', 'name', 'subject', 'email', 'message']

    def has_add_permission(self, request):
        return False

    def change_view(self, request, object_id, form_url='', extra_context=None):
        try:
            feedback_object = Feedback.objects.get(id=object_id)
            if not feedback_object.is_read:
                feedback_object.is_read = True
                feedback_object.save()
        except ObjectDoesNotExist:
            pass

        return super(FeedbackAdmin, self).change_view(request, object_id, form_url=form_url, extra_context=extra_context)

@admin.register(Seo)
class SeoAdmin(admin.ModelAdmin):

    fieldsets = (
        ('Главная страница', {'fields': (
            'index_page_title',
            'index_page_meta_keywords',
            'index_page_meta_description',
            'index_page_meta_og_title',
            'index_page_meta_og_description',
            'index_page_meta_og_image',
        )}),
        ('Новости (Каталог)', {'fields': (
            'blog_list_page_title',
            'blog_list_page_meta_keywords',
            'blog_list_page_meta_description',
            'blog_list_page_meta_og_title',
            'blog_list_page_meta_og_description',
            'blog_list_page_meta_og_image',
        )}),
        ('Гранд Смета (Каталог)', {'fields': (
            'grand_smeta_list_page_title',
            'grand_smeta_list_page_meta_keywords',
            'grand_smeta_list_page_meta_description',
            'grand_smeta_list_page_meta_og_title',
            'grand_smeta_list_page_meta_og_description',
            'grand_smeta_list_page_meta_og_image',
        )}),
        ('1C (Каталог)', {'fields': (
            'one_c_list_page_title',
            'one_c_list_page_meta_keywords',
            'one_c_list_page_meta_description',
            'one_c_list_page_meta_og_title',
            'one_c_list_page_meta_og_description',
            'one_c_list_page_meta_og_image',
        )}),
        ('Онлайн Кассы (Каталог)', {'fields': (
            'online_cashbox_list_page_title',
            'online_cashbox_list_page_meta_keywords',
            'online_cashbox_list_page_meta_description',
            'online_cashbox_list_page_meta_og_title',
            'online_cashbox_list_page_meta_og_description',
            'online_cashbox_list_page_meta_og_image',
        )}),
        ('Таблица с расписанием', {'fields': (
            'schedule_table_page_title',
            'schedule_table_page_meta_keywords',
            'schedule_table_page_meta_description',
            'schedule_table_page_meta_og_title',
            'schedule_table_page_meta_og_description',
            'schedule_table_page_meta_og_image',
        )}),
        ('Каталог курсов', {'fields': (
            'schedule_list_page_title',
            'schedule_list_page_meta_keywords',
            'schedule_list_page_meta_description',
            'schedule_list_page_meta_og_title',
            'schedule_list_page_meta_og_description',
            'schedule_list_page_meta_og_image',
        )}),
        ('Контакты', {'fields': (
            'contacts_page_title',
            'contacts_page_meta_keywords',
            'contacts_page_meta_description',
            'contacts_page_meta_og_title',
            'contacts_page_meta_og_description',
            'contacts_page_meta_og_image',
        )}),
        ('Лицензии', {'fields': (
            'licenses_page_title',
            'licenses_page_meta_keywords',
            'licenses_page_meta_description',
            'licenses_page_meta_og_title',
            'licenses_page_meta_og_description',
            'licenses_page_meta_og_image',
        )}),
        ('Преподователи', {'fields': (
            'educators_page_title',
            'educators_page_meta_keywords',
            'educators_page_meta_description',
            'educators_page_meta_og_title',
            'educators_page_meta_og_description',
            'educators_page_meta_og_image',
        )})
    )

    def has_add_permission(self, request):
        return self.model.objects.count() < 1

@admin.register(PrivacyPolicy)
class PrivacyPolicyAdmin(admin.ModelAdmin):
    fieldsets = (
        ('Общее', {'fields': (
            'description',
        )}),
        ('SEO', {'fields': (
            'page_title',
            'page_meta_keywords',
            'page_meta_description',
            'page_meta_og_title',
            'page_meta_og_description',
            'page_meta_og_image',
        )})
    )

    def has_add_permission(self, request):
        return self.model.objects.count() < 1

@admin.register(PromoVisitor)
class PromoVisitorAdmin(admin.ModelAdmin):
    readonly_fields = ['get_post_display', 'user', 'ip']

    fields = ('form_data', 'get_post_display', 'user', 'ip')

    class Media:
        js = ('js/admin/promo.form.viewer.js',)
        css = {
            'all': ('js/admin/promo.form.viewer.css',)
        }

    def has_add_permission(self, request):
        return False

class PromoFormFieldInline(admin.StackedInline):
    model = PromoFormField
    extra = 1

@admin.register(PromoFormBuilder)
class PromoFormBuilderAdmin(admin.ModelAdmin):
    inlines = (PromoFormFieldInline,)

    exclude = ('captcha',)

    def has_add_permission(self, request):
        return self.model.objects.count() < 1


from django.contrib import admin
from django.urls import path, include

from apps.health.views import healthcheck
import apps.vending.views as vending_views

urlpatterns = [
    path('admin/', admin.site.urls),
    path("healthcheck/", healthcheck),
    path("slots/", include([
        path("", vending_views.VendingMachineSlotView.as_view()),
    ])),
    path("clients/", include([
        path("", vending_views.ClientView.as_view()),
    ])),
]


import logging
from typing import Any, Mapping, Optional, Sequence
from uuid import uuid4

from django.conf import settings
from django.utils import timezone
from rest_framework import serializers

from sentry.auth.access import SystemAccess
from sentry.incidents.logic import (
    ChannelLookupTimeoutError,
    InvalidTriggerActionError,
    get_slack_channel_ids,
)
from sentry.incidents.models import AlertRule, AlertRuleTriggerAction
from sentry.incidents.serializers import AlertRuleSerializer
from sentry.integrations.slack.client import SlackClient
from sentry.integrations.utils import get_identities_by_user
from sentry.mediators import project_rules
from sentry.models import (
    Identity,
    IdentityProvider,
    IdentityStatus,
    Integration,
    Organization,
    Project,
    Rule,
    RuleActivity,
    RuleActivityType,
    User,
    UserEmail,
)
from sentry.shared_integrations.exceptions import (
    ApiError,
    ApiRateLimitedError,
    DuplicateDisplayNameError,
)
from sentry.tasks.base import instrumented_task
from sentry.utils import json
from sentry.utils.json import JSONData
from sentry.utils.redis import redis_clusters

from .utils import (
    SLACK_RATE_LIMITED_MESSAGE,
    get_channel_id_with_timeout,
    get_slack_data_by_user,
    strip_channel_name,
)

logger = logging.getLogger("sentry.integrations.slack.tasks")

SLACK_FAILED_MESSAGE = (
    "The slack resource does not exist or has not been granted access in that workspace."
)

class RedisRuleStatus:
    def __init__(self, uuid: Optional[str] = None) -> None:
        self._uuid = uuid or self._generate_uuid()

        cluster_id = getattr(settings, "SENTRY_RULE_TASK_REDIS_CLUSTER", "default")
        self.client = redis_clusters.get(cluster_id)
        self._set_initial_value()

    @property
    def uuid(self) -> str:
        return self._uuid

    def set_value(
        self, status: str, rule_id: Optional[int] = None, error_message: Optional[str] = None
    ) -> None:
        value = self._format_value(status, rule_id, error_message)
        self.client.set(self._get_redis_key(), f"{value}", ex=60 * 60)

    def get_value(self) -> JSONData:
        key = self._get_redis_key()
        value = self.client.get(key)
        return json.loads(value)

    def _generate_uuid(self) -> str:
        return uuid4().hex

    def _set_initial_value(self) -> None:
        value = json.dumps({"status": "pending"})
        self.client.set(self._get_redis_key(), f"{value}", ex=60 * 60, nx=True)

    def _get_redis_key(self) -> str:
        return f"slack-channel-task:1:{self.uuid}"

    def _format_value(
        self, status: str, rule_id: Optional[int], error_message: Optional[str]
    ) -> str:
        value = {"status": status}
        if rule_id:
            value["rule_id"] = str(rule_id)
        if error_message:
            value["error"] = error_message
        elif status == "failed":
            value["error"] = SLACK_FAILED_MESSAGE

        # Explicitly typing to satisfy mypy.
        _value: str = json.dumps(value)
        return _value

@instrumented_task(name="sentry.integrations.slack.search_channel_id", queue="integrations")  # type: ignore
def find_channel_id_for_rule(
    project: Project,
    actions: Sequence[AlertRuleTriggerAction],
    uuid: str,
    rule_id: Optional[int] = None,
    user_id: Optional[int] = None,
    **kwargs: Any,
) -> None:
    redis_rule_status = RedisRuleStatus(uuid)

    try:
        project = Project.objects.get(id=project.id)
    except Project.DoesNotExist:
        redis_rule_status.set_value("failed")
        return

    user = None
    if user_id:
        try:
            user = User.objects.get(id=user_id)
        except User.DoesNotExist:
            pass

    organization = project.organization
    integration_id: Optional[int] = None
    channel_name: Optional[str] = None

    # TODO: make work for multiple Slack actions
    for action in actions:
        if action.get("workspace") and action.get("channel"):
            integration_id = action["workspace"]
            # we need to strip the prefix when searching on the channel name
            channel_name = strip_channel_name(action["channel"])
            break

    try:
        integration = Integration.objects.get(
            provider="slack", organizations=organization, id=integration_id
        )
    except Integration.DoesNotExist:
        redis_rule_status.set_value("failed")
        return

    # We do not know exactly how long it will take to paginate through all of the Slack
    # endpoints but need some time limit imposed. 3 minutes should be more than enough time,
    # we can always update later
    try:
        (prefix, item_id, _timed_out) = get_channel_id_with_timeout(
            integration, channel_name, 3 * 60
        )
    except DuplicateDisplayNameError:
        # if we find a duplicate display name and nothing else, we
        # want to set the status to failed. This just lets us skip
        # over the next block and hit the failed status at the end.
        item_id = None
        prefix = ""
    except ApiRateLimitedError:
        redis_rule_status.set_value("failed", None, SLACK_RATE_LIMITED_MESSAGE)
        return

    if item_id:
        for action in actions:
            # need to make sure we are adding back the right prefix and also the channel_id
            if action.get("channel") and strip_channel_name(action.get("channel")) == channel_name:
                action["channel"] = prefix + channel_name
                action["channel_id"] = item_id
                break

        kwargs["actions"] = actions
        kwargs["project"] = project

        if rule_id:
            rule = Rule.objects.get(id=rule_id)
            rule = project_rules.Updater.run(rule=rule, pending_save=False, **kwargs)
        else:
            rule = project_rules.Creator.run(pending_save=False, **kwargs)
            if user:
                RuleActivity.objects.create(
                    rule=rule, user=user, type=RuleActivityType.CREATED.value
                )

        redis_rule_status.set_value("success", rule.id)
        return
    # if we never find the channel name we failed :(
    redis_rule_status.set_value("failed")

@instrumented_task(  # type: ignore
    name="sentry.integrations.slack.search_channel_id_metric_alerts", queue="integrations"
)
def find_channel_id_for_alert_rule(
    organization_id: int,
    uuid: str,
    data: Any,
    alert_rule_id: Optional[int] = None,
    user_id: Optional[int] = None,
) -> None:
    redis_rule_status = RedisRuleStatus(uuid)
    try:
        organization = Organization.objects.get(id=organization_id)
    except Organization.DoesNotExist:
        redis_rule_status.set_value("failed")
        return

    user = None
    if user_id:
        try:
            user = User.objects.get(id=user_id)
        except User.DoesNotExist:
            pass

    alert_rule = None
    if alert_rule_id:
        try:
            alert_rule = AlertRule.objects.get(organization_id=organization_id, id=alert_rule_id)
        except AlertRule.DoesNotExist:
            redis_rule_status.set_value("failed")
            return

    try:
        mapped_ids = get_slack_channel_ids(organization, user, data)
    except (serializers.ValidationError, ChannelLookupTimeoutError, InvalidTriggerActionError) as e:
        # channel doesn't exist error or validation error
        logger.info(
            "get_slack_channel_ids.failed",
            extra={
                "exception": e,
            },
        )
        redis_rule_status.set_value("failed")
        return
    except ApiRateLimitedError as e:
        logger.info(
            "get_slack_channel_ids.rate_limited",
            extra={
                "exception": e,
            },
        )
        redis_rule_status.set_value("failed", None, SLACK_RATE_LIMITED_MESSAGE)
        return

    for trigger in data["triggers"]:
        for action in trigger["actions"]:
            if action["type"] == "slack":
                if action["targetIdentifier"] in mapped_ids:
                    action["input_channel_id"] = mapped_ids[action["targetIdentifier"]]
                else:
                    # We can early exit because we couldn't map this action's slack channel name to a slack id
                    # This is a fail safe, but I think we shouldn't really hit this.
                    redis_rule_status.set_value("failed")
                    return

    # we use SystemAccess here because we can't pass the access instance from the request into the task
    # this means at this point we won't raise any validation errors associated with permissions
    # however, we should only be calling this task after we tried saving the alert rule first
    # which will catch those kinds of validation errors
    serializer = AlertRuleSerializer(
        context={
            "organization": organization,
            "access": SystemAccess(),
            "user": user,
            "use_async_lookup": True,
            "validate_channel_id": False,
        },
        data=data,
        instance=alert_rule,
    )
    if serializer.is_valid():
        try:
            alert_rule = serializer.save()
            redis_rule_status.set_value("success", alert_rule.id)
            return
        # we can still get a validation error for the channel not existing
        except (serializers.ValidationError, ChannelLookupTimeoutError):
            # channel doesn't exist error or validation error
            redis_rule_status.set_value("failed")
            return
    # some other error
    redis_rule_status.set_value("failed")
    return

@instrumented_task(name="sentry.integrations.slack.link_users_identities", queue="integrations")  # type: ignore
def link_slack_user_identities(integration: Integration, organization: Organization) -> None:
    emails_by_user = UserEmail.objects.get_emails_by_user(organization)
    slack_data_by_user = get_slack_data_by_user(integration, organization, emails_by_user)

    idp = IdentityProvider.objects.get(
        type=integration.provider,
        external_id=integration.external_id,
    )
    date_verified = timezone.now()
    identities_by_user = get_identities_by_user(idp, slack_data_by_user.keys())

    for user, data in slack_data_by_user.items():
        # Identity already exists, the emails match, AND the external ID has changed
        if user in identities_by_user.keys():
            if data["slack_id"] != identities_by_user[user].external_id:
                # replace the Identity's external_id with the new one we just got from Slack
                identities_by_user[user].update(external_id=data["slack_id"])
            continue
        # the user doesn't already have an Identity and one of their Sentry emails matches their Slack email
        matched_identity, created = Identity.objects.get_or_create(
            idp=idp,
            external_id=data["slack_id"],
            defaults={"user": user, "status": IdentityStatus.VALID, "date_verified": date_verified},
        )
        # the Identity matching that idp/external_id combo is linked to a different user
        if not created:
            logger.info(
                "post_install.identity_linked_different_user",
                extra={
                    "idp_id": idp.id,
                    "external_id": data["slack_id"],
                    "object_id": matched_identity.id,
                    "user_id": user.id,
                    "type": idp.type,
                },
            )

# TODO: add retry logic
@instrumented_task(name="sentry.integrations.slack.post_message", queue="integrations", max_retries=0)  # type: ignore
def post_message(
    payload: Mapping[str, Any], log_error_message: str, log_params: Mapping[str, Any]
) -> None:
    client = SlackClient()
    try:
        client.post("/chat.postMessage", data=payload, timeout=5)
    except ApiError as e:
        extra = {"error": str(e), **log_params}
        logger.info(log_error_message, extra=extra)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Apr  2 14:37:24 2017

@author: gjacopo
"""

from __future__ import print_function

import warnings
from collections import OrderedDict

import requests

try:
    import numpy as np#analysis:ignore
except:
    raise IOError("Package numpy not imported - Requested")

try:
    import pandas as pd
    PDVERS = int(pd.__version__.split('.')[1])
except:
    PDVERS = 0 # unknown
    raise IOError("Package pandas not imported - Requested")

try:
    import json
except:
    try:
        import simplejson as json#analysis:ignore
    except:
        warnings.warn("Package simplejson/json not imported")

try:
    #import pyjstat
    from pyjstat import pyjstat
except:
    warnings.warn("Package pyjstat not imported - Used for JSONSTAT format")

__GEO_LABELS  = [('EU28', 'European Union (28 countries)'),
               # ('EU27':, 'European Union (27 countries)'),
               # ('EA19', 'Euro area (19 countries)'),
               # ('EA18', 'Euro area (18 countries)'),
               # ('EU', 'European Union (EU6-1972, EU9-1980, EU10-1985, EU12-1994, EU15-2004, EU25-2006, EU27-2013, EU28)'),
               # ('EA', 'Euro area (EA11-2000, EA12-2006, EA13-2007, EA15-2008, EA16-2010, EA17-2013, EA18-2014, EA19)'),
               ('AT', 'Austria'),
               ('BE', 'Belgium'),
               ('BG', 'Bulgaria'),
               ('CY', 'Cyprus'),
               ('CZ', 'Czech Republic'),
               ('DE', 'Germany (until 1990 former territory of the FRG)'),
               ('DK', 'Denmark'),
               ('EE', 'Estonia'),
               ('EL', 'Greece'),
               ('ES', 'Spain'),
               ('FI', 'Finland'),
               ('FR', 'France'),
               ('HR', 'Croatia'),
               ('HU', 'Hungary'),
               ('IE', 'Ireland'),
               ('IT', 'Italy'),
               ('LT', 'Lithuania'),
               ('LU', 'Luxembourg'),
               ('LV', 'Latvia'),
               ('MT', 'Malta'),
               ('NL', 'Netherlands'),
               ('PL', 'Poland'),
               ('PT', 'Portugal'),
               ('RO', 'Romania'),
               ('SE', 'Sweden'),
               ('SI', 'Slovenia'),
               ('SK', 'Slovakia'),
               ('UK', 'United Kingdom'),
               ('CH', 'Switzerland'),
               ('NO', 'Norway'),
               ('IS', 'Iceland'),
               ('MK', 'the Former Yugoslav Republic of Macedonia'),
               ('RS', 'Serbia'),
               ('TR', 'Turkey')]
GEO_LABELS  = OrderedDict(__GEO_LABELS)

__AGE_LABELS = [('TOTAL', 'Total population'),
                ('Y_LT6', 'Less than 6 years'),
                ('Y6-10', 'From 6 to 10 years'),
                ('Y11-15', 'From 11 to 15 years'),
                ('Y_LT16', 'Less than 16 years'),
                ('Y16-19', 'From 16 to 19 years'),
                ('Y16-24', 'From 16 to 24 years'),
                ('Y_GE18', '18 years or over'),
                ('Y18-24', 'From 18 to 24 years'),
                ('Y20-24', 'From 20 to 24 years'),
                ('Y25-29', 'From 25 to 29 years'),
                ('Y25-34', 'From 25 to 34 years'),
                ('Y25-49', 'From 25 to 49 years'),
                ('Y35-44', 'From 35 to 44'),
                ('Y45-54', 'From 45 to 54'),
                ('Y50-64', 'From 50 to 64 years'),
                ('Y55-64', 'From 55 to 64 years'),
                ('Y65-74', 'From 65 to 74 years'),
                ('Y_GE75', '75 years or over')
                ]

AGE_LABELS  = OrderedDict(__AGE_LABELS)

SEX_LABELS  = {'M': 'Males',
               'F': 'Females',
               'T': 'Total'}

UNIT_LABELS = {'THS_PER': 'Thousand persons',
               'PC_POP': 'Percentage of total population'
               }

STATUS_LABELS = {'POP': 'Population',
                'EMP': 'Employed persons',
                'SAL': 'Employees',
                'NSAL': 'Employed persons except employees',
                'NEMP': 'Not employed persons',
                'UNE': 'Unemployed persons',
                'RET': 'Retired persons',
                'INAC_OTH': 'Other inactive persons'}

class EstatDataFrame(object):

    PROTOCOL        = "http"
    API_LANG        = "en"
    API_FMT         = "json"
    API_DOMAIN      = 'ec.europa.eu/eurostat/wdds'
    API_VERS        = 2.1
    API_URL         = "{}://{}/rest/data/v{}/{}/{}".format(
                  PROTOCOL, API_DOMAIN, API_VERS, API_FMT, API_LANG
                  )

    def __init__(self, **kwargs):
        self.__indicator = []
        self.__url = ""
        self.__response = None
        if kwargs == {}:
            return
        if 'indicator' in kwargs:
            self.__indicator = kwargs.pop('indicator')

    @property
    def indicator(self):
        return self.__indicator

    @property
    def url(self):
        return self.__url

    @property
    def response(self):
        return self.__response

    def build_url(self, **kwargs):
        url = "{}/{}?".format(EstatDataFrame.API_URL, self.indicator[0])
        #if 'geo' in kwargs:
        #    url = "{}geo={}&".format(url, kwargs.pop('geo', None))
        _izip_replicate = lambda d : [(k,i) if isinstance(d[k], (tuple,list))        \
                else (k, d[k]) for k in d for i in d[k]]
        #_no_replicate = lambda d : d.items()
        filters = '&'.join(['{k}={v}'.format(k=k, v=v) for (k, v) in _izip_replicate(kwargs)])
        try:
            self.__url = "{}{}".format(url, filters)
        except:
            pass
        return self.url

    def get_response(self):
        # request the URL
        session = requests.session()
        try:
            response = session.head(self.url)
            response.raise_for_status()
        except:
            raise IOError("ERROR: wrong request formulated")
        else:
            print ("OK: status={}".format(response.status_code))
        # load the data
        try:
            response = session.get(self.url)
        except:
            raise IOError('error retrieveing response from URL')
        try:
            if EstatDataFrame.API_FMT == 'json':
                self.__response = response.json()
            elif EstatDataFrame.API_FMT == 'unicode':
                self.__response = response.text
        except:
            pass
        return self.response

    def load_json(self, pivot=None, index=None, use_label=True):
        try:
            # pyjstat produces dataframe in stacked shape
            dataset = pyjstat.Dataset.read(self.url)
        except:
            raise IOError("Dataframe not created")
        df = dataset.write('dataframe')
        if use_label is True:
            for name in dataset['dimension']:
                dname = dataset['dimension'][name]['category']['label']
                df[name] = df[name].map(dict((v,k) for k,v in dname.items()))
        if pivot is None:
            return df
        if index is None:
            index = set(df.columns).difference({pivot}.union({'value'}))
            # or: index = [c for c in df.columns if c not in {pivot, 'ivalue'}]
        if PDVERS < 18:
            df = df.set_index(index + [pivot]).unstack(pivot)
        else:
            df = df.pivot_table(index=index, columns=pivot, values='value')
        df.reset_index(drop=False, inplace=True)
        try:
            #df.index.name = None
            #df = df.reindex(df.index.rename(None))
            df.columns.name = None
        except:
            pass
        return df

    def __call__(self, **kwargs):
        # input data loading and formatting
        url = self.build_url(**kwargs)#analysis:ignore
        resp = self.get_response()#analysis:ignore
        df = self.load_json(pivot='time')
        return df