import json
import configparser
import os
import requests
from pathlib import Path
import sqlite3
import pandas
from math import ceil
import datetime

class SearchAnalyzer():
    session = requests.Session()

    __BASE_API_URL = 'https://line.pr-cy.ru/api/v1.1.0'
    low_price = 0.25
    high_price = 0.3

    __tasks_resaults = []

    def __init__(self, *, key: str = None, use_config: bool = True, search_engine: str, search_type: str, search_depth: str, file_path: str, search_region: int = 1) -> None:
        if use_config:
            config = configparser.ConfigParser()
            config.read(os.path.join('cfg', 'config.ini'))
            self.__API_KEY = config['DEFAULT']['API_KEY']
            self.low_price = config['DEFAULT']['low_price']
            self.height_price = config['DEFAULT']['high_price']
            if search_engine == 'yandex':
                self.region_id = config['DEFAULT']['REGION_ID_Yandex']
            if search_engine == 'google':
                self.region_id = config['DEFAULT']['REGION_ID_Google']

        else:
            if not key:
                raise ValueError(
                    'Должен быть указан API ключ при инициализации класа, либо же использоваться use_config = True')
            else:
                self.__API_KEY = key

        self.search_engine = search_engine
        self.search_type = search_type
        self.search_depth = search_depth
        if Path(file_path).is_file():
            self.query_file_path = file_path
        else:
            raise FileExistsError('Файл не найден!')

    def check_balanc(self, keywords):
        balance = self.get_balance(self.session)
        if len(keywords) > 100:
            if (balance - (len(keywords)*self.low_price)) > 0:
                print(
                    f'Баланс составляет: {balance}, стоимость текущего задания ~ { len(keywords)*self.low_price }')
                return True
            else:
                print(
                    f'Баланс составляет: {balance}, стоимость текущего задания ~ { len(keywords)*self.low_price }')
                return False

        if len(keywords) <= 100:
            if (balance - (len(keywords)*self.high_price)) > 0:
                print(
                    f'Баланс составляет: {balance}, стоимость текущего задания ~ { len(keywords)*self.high_price }')
                return True
            else:
                print(
                    f'Баланс составляет: {balance}, стоимость текущего задания ~ { len(keywords)*self.high_price }')
                return False

    def get_balance(self,):
        response = self.session.get(
            self.__BASE_API_URL+'/user/balance', params={'key': self.__API_KEY})
        if response.status_code == 200:
            return json.loads(response.text)['balance']
        else:
            print(f"Вохникла ошибка: {json.loads(response.text).get('error')}")
            return None

    def creat_task(self, keywords):
        """Принимает до 1000 ключевых запросов"""
        if len(keywords) >=1000:
            raise ValueError('Передано слишком много ключей на проверку')

        response = self.session.post(self.__BASE_API_URL+'/task/create', params = {'key' : self.__API_KEY},
                    data = {
                    'keywords[]': keywords,
                    'engine' : self.search_engine,
                    'regionId' : self.region_id,
                    'device' : self.search_type,
                    'language' : 'ru'
                    })
        if response.status_code == 200:
            taskId = json.loads(response.text)['taskId']
            return taskId
        else:
            raise Exception(f'Ошибка создания задачи\n{response.status_code}\n{json.loads(response.text)}')

    def task_status_check(self, taskId):
        response = self.session.get(self.__BASE_API_URL+ f'/task/status/{taskId}', params = {'key': self.__API_KEY})
        if response.status_code == 200:
            status = json.loads(response.text)['status']
            if status == 'done':
                return True
            else:
                print(f"Статус задачи {taskId}: {status}")
                return False
        else:
            raise Exception(f'Ошибка проверки статуса задачи\n{response.status_code}\n{json.loads(response.text)}')

    def get_keywords_from_file(self) -> list:
        """return : Возращает список со списками ключевых слов из файла с количеством ключевиков до 1000 шт"""
        df = pandas.read_excel(self.query_file_path, header=None)
        data = df.iloc[ : , 0].tolist()
        self.keywords_count = len(data)
        self.keywords = []
        if len(data) >=1000:
            n = ceil( len(data)/ ceil(len(data)/999))
            for x in range(0, len(data), n):
                e_c = data[x : n + x]
                self.keywords.append(e_c)
        else:
            self.keywords.append(data)
        return self.keywords

    def get_task_resault(self, taskId):
        response = self.session.get(self.__BASE_API_URL+f'/task/result/{taskId}', params = {'key':self.__API_KEY})
        if response.status_code == 200:
            result = json.loads(response.text)
            self.__tasks_resaults.append(result)
            return result
        else:
            raise ConnectionError(f'Не удалось получить результаты {taskId}')

    @property
    def tasks_resaults(self):
        return [item for item in self.__tasks_resaults]

    def get_distinct_hosts(self):
        distinct_hosts = []
        for task_resault in self.__tasks_resaults:
            for keyword in task_resault['keywords']:
                for resault in keyword['serp'][:self.search_depth]:
                    host = resault['host'].replace('www.', '').lower()
                    if host not in distinct_hosts:
                        distinct_hosts.append(host)
        return distinct_hosts

    def get_query_resaults(self):
        self.final_data = []
        for task_resaults in self.__tasks_resaults:
            for keyword in task_resaults['keywords']:
                for resault in keyword['serp'][:self.search_depth]:
                    res={}
                    res['query'] = keyword['query']
                    res['position'] = resault['position']
                    res['url'] = resault['url']
                    res['host'] = resault['host']
                    res['title'] = resault['title']
                    self.final_data.append(res)

        return self.final_data

    def get_distinct_hosts_statistic(self, ):
        result = []
        for distinct_host in self.get_distinct_hosts():
            counter = 0
            for task_resaults in self.tasks_resaults:
                for keyword in task_resaults['keywords']:
                    for resault in keyword['serp'][:self.search_depth]:
                        if distinct_host.lower() in resault['host'].lower():
                            counter +=1
                            break;
            result.append(
                (distinct_host.lower(), round((counter / self.keywords_count)*100, 2))
            )
        return result

    def get_distinct_hosts_statistic_top10(self, ):
        result = []
        for distinct_host in self.get_distinct_hosts():
            counter = 0
            for task_resaults in self.tasks_resaults:
                for keyword in task_resaults['keywords']:
                    for resault in keyword['serp'][:10]:
                        if distinct_host.lower() in resault['host'].lower():
                            counter +=1
                            break;
            result.append(
                (distinct_host.lower(), round((counter / self.keywords_count)*100, 2))
            )
        return result

class DomainAnalyzer():

    session = requests.Session()

    __BASE_API_URL = 'https://apis.pr-cy.ru/api/v1.1.0'
    __tasks_resaults = []

    api_query_count = 0

    def __init__(self, *, key: str = None, use_config: bool = True, file_path: str = None ) -> None:
        if use_config:
            config = configparser.ConfigParser()
            config.read(os.path.join('cfg', 'config.ini'))
            self.__API_KEY = config['DEFAULT']['DOMAIN_API_KEY']
            self.google_spreadsheet_url = config['DEFAULT']['Google_spreadsheet_url'] + '/export?format=xlsx'
            self.FRESHNESS = int(config['DEFAULT']['FRESHNESS_days'])

        else:
            if not key:
                raise ValueError(
                    'Должен быть указан API ключ при инициализации класа, либо же использоваться use_config = True')
            else:
                self.__API_KEY = key

        if Path(file_path).is_file():
            self.query_file_path = file_path
        else:
            raise FileExistsError('Файл не найден!')

    @property
    def tasks_resaults(self):
        return self.__tasks_resaults

    def get_domains_from_file(self) -> list:
        df = pandas.read_excel(self.query_file_path, header=None)
        data = df.iloc[ : , 0].tolist()
        self.domains_count = len(data)
        self.domains = data
        if self.domains_count >= 200:
            print(f'ПРЕДУПРЕЖДЕНИЕ! Количество доменов для анализа привышает 200 ({self.domains_count}), \nОграничение api: 200 запросов в час, обработка может занять много времени')
        return data

    def get_data_from_google_sheet(self):
        # with sqlite3.connect('local.db') as connection:
            # create a cursor object from the cursor class
            # cur = connection.cursor()
        df = pandas.read_excel(self.google_spreadsheet_url, skiprows=1)
        return df

    def update_domain_analysis(self, domain):
        url = f'/analysis/update/base/{domain}'
        response = requests.post(self.__BASE_API_URL+url, params={'key':self.__API_KEY})
        self.api_query_count+=1
        if response.status_code != 200:
            print(f'Ошибка при попытке отправки запроса на обновление {domain}')
            return False

    def get_domain_analysis(self, domain):
        url = f'/analysis/base/{domain}'
        response = requests.get(self.__BASE_API_URL+url,
                                params={
                                            'key': self.__API_KEY,
                                            'tests':'avgVisitDuration,googleIndex,ip,mainPageExternalLinks,megaindexLinksCount,pageSpeedDesktop,pageSpeedMobile,pagesPerVisit,publicStatistics,sitemap,trafficGeography,trafficSources,whoisCreationDate,yandexAchievements,yandexIndex,yandexReviews',
                                            'excludeHistory': 1
                                        }
                                )
        self.api_query_count+=1
        if response.status_code == 429:
            return False
        data = json.loads(response.text)
        data['host'] = domain
        self.__tasks_resaults.append(data)
        return data

    def check_domain_status(self, domain):
        url = f'/analysis/status/base/{domain}'
        response = requests.get(self.__BASE_API_URL+url, params={'key': self.__API_KEY})
        response_content = json.loads(response.text)
        if response.status_code != 200:
            return (False, 'new')
        if not response_content.get('updated'):
            return (False, 'updating')
        if datetime.datetime.strptime(response_content.get('updated'), "%Y-%m-%dT%H:%M:%S+03:00") >= datetime.datetime.now()- datetime.timedelta(days=self.FRESHNESS) and \
                not response_content.get('isUpdating'):
            return (True, 'Ок')
        if datetime.datetime.strptime(response_content.get('updated'), "%Y-%m-%dT%H:%M:%S+03:00") <= datetime.datetime.now()- datetime.timedelta(days=self.FRESHNESS):
            return (False, 'overdue')
        if response_content.get('isUpdating'):
            return (False, 'updating')

    def get_final_records(self):
        google_df = self.get_data_from_google_sheet()
        final_data = []
        for host_info in self.__tasks_resaults:

            res={}
            res['host'] = host_info['host']
            try: res['googleIndex'] = host_info['googleIndex']['googleIndex']
            except: res['googleIndex'] = ''
            try: res['yandexIndex'] = host_info['yandexIndex']['yandexIndex']
            except: res['yandexIndex'] = ''
            try: res['Длина визита, сек'] = host_info['avgVisitDuration']['avgVisitDuration']
            except: res['Длина визита, сек'] = ''
            try: res['ip'] = host_info['ip']['ip']
            except: res['ip'] = ''
            try: res['Кол-во внешних ссылок по Мегаиндекс'] = host_info['megaindexLinksCount']['megaindexLinksCount']
            except: res['Кол-во внешних ссылок по Мегаиндекс'] = ''
            try: res['ПК pageSpeed оценка'] = host_info['pageSpeedDesktop']['pageSpeed']['score']
            except: res['ПК pageSpeed оценка'] = ''
            try: res['ПК pageSpeed в секундах'] = host_info['pageSpeedDesktop']['pageSpeed']['value']
            except:  res['ПК pageSpeed в секундах'] =''
            try: res['Мобильный pageSpeed оценка'] = host_info['pageSpeedMobile']['pageSpeed']['score']
            except:  res['Мобильный pageSpeed оценка']  = ''
            try: res['Мобильный pageSpeed в секундах'] = host_info['pageSpeedMobile']['pageSpeed']['value']
            except: res['Мобильный pageSpeed в секундах'] = ''
            try: res['Просмотры в мес'] = host_info['publicStatistics']['publicStatisticsPageViewsMonthly']
            except: res['Просмотры в мес'] = ''
            try: res['Посетители в мес'] = host_info['publicStatistics']['publicStatisticsPrcyVisitsMonthly']
            except: res['Посетители в мес'] = ''
            try: res['sitemap'] = host_info['sitemap']['sitemapUrl']
            except: res['sitemap'] = ''
            try: res['Трафик по странам'] = host_info['trafficGeography']['topCountryGeography']
            except: res['Трафик по странам'] = ''
            try: res['Ист. траф. Прямые заходы'] = host_info['trafficSources']['trafficSourcesDirect']/100
            except: res['Ист. траф. Прямые заходы'] = ''
            try: res['Ист. траф. Почтовые рассылки'] = host_info['trafficSources']['trafficSourcesMail']/100
            except: res['Ист. траф. Почтовые рассылки'] = ''
            try: res['Ист. траф. Ссылки на сайтах'] = host_info['trafficSources']['trafficSourcesReferrals']/100
            except:  res['Ист. траф. Ссылки на сайтах'] =''
            try: res['Ист. траф. Поисковые системы'] = host_info['trafficSources']['trafficSourcesSearch']/100
            except: res['Ист. траф. Поисковые системы'] = ''
            try: res['Ист. траф. Соц сети'] = host_info['trafficSources']['trafficSourcesSocial']/100
            except: res['Ист. траф. Соц сети'] = ''
            try: res['Дата создания сайта'] = datetime.datetime.strptime(host_info['whoisCreationDate']['whoisCreationDate'], '%Y-%m-%dT%H:%M:%S+03:00')
            except: res['Дата создания сайта'] = ''
            try: res['Сайт на https?'] = host_info['yandexAchievements']['yandexAchievementsHttps']
            except: res['Сайт на https?']  = ''
            try: res['Есть Турбо?'] = host_info['yandexAchievements']['yandexAchievementsTurbo']
            except: res['Есть Турбо?'] = ''
            try: res['Отзывов в Яндекс'] = host_info['yandexReviews']['count']
            except: res['Отзывов в Яндекс'] = ''
            try: res['Внешних ссылок на главную'] = host_info['mainPageExternalLinks']['externalIndexCount']
            except: res['Внешних ссылок на главную'] = ''
            try: res['Наш партнер?'] = google_df.loc[google_df['Домен'] == host_info['host']].iloc[0]['Наш партнер?']
            except: res['Наш партнер?'] = ''
            try:res['Ответственный менеджер'] = google_df.loc[google_df['Домен'] == host_info['host']].iloc[0]['Ответственный менеджер']
            except: res['Ответственный менеджер'] = ''
            try:res['Тип партнера'] = google_df.loc[google_df['Домен'] == host_info['host']].iloc[0]['Тип партнера']
            except:res['Тип партнера'] =''

            final_data.append(res)

        return final_data

class SearchRepeater():
    __tasks_resaults = []
    __BASE_API_URL = 'https://line.pr-cy.ru/api/v1.1.0'
    session = requests.Session()

    def __init__(self, search_depth, keywords_count):
        config = configparser.ConfigParser()
        config.read(os.path.join('cfg', 'config.ini'))
        self.__API_KEY = config['DEFAULT']['API_KEY']
        self.search_depth = search_depth

        self.keywords_count = keywords_count

    def task_status_check(self, taskId):
        response = self.session.get(self.__BASE_API_URL+ f'/task/status/{taskId}', params = {'key': self.__API_KEY})
        if response.status_code == 200:
            status = json.loads(response.text)['status']
            if status == 'done':
                return True
            else:
                print(f"Статус задачи {taskId}: {status}")
                return False
        else:
            raise Exception(f'Ошибка проверки статуса задачи\n{response.status_code}\n{json.loads(response.text)}')

    @property
    def tasks_resaults(self):
        return [item for item in self.__tasks_resaults]

    def get_distinct_hosts(self):
        distinct_hosts = []
        for task_resault in self.__tasks_resaults:
            for keyword in task_resault['keywords']:
                for resault in keyword['serp'][:self.search_depth]:
                    host = resault['host'].replace('www.', '').lower()
                    if host not in distinct_hosts:
                        distinct_hosts.append(host)
        return distinct_hosts

    def get_query_resaults(self):
        self.final_data = []
        for task_resaults in self.__tasks_resaults:
            for keyword in task_resaults['keywords']:
                for resault in keyword['serp'][:self.search_depth]:
                    res={}
                    res['query'] = keyword['query']
                    res['position'] = resault['position']
                    res['url'] = resault['url']
                    res['host'] = resault['host']
                    res['title'] = resault['title']
                    self.final_data.append(res)

        return self.final_data

    def get_distinct_hosts_statistic(self, ):
        result = []
        for distinct_host in self.get_distinct_hosts():
            counter = 0
            for task_resaults in self.tasks_resaults:
                for keyword in task_resaults['keywords']:
                    for resault in keyword['serp'][:self.search_depth]:
                        if distinct_host.lower() in resault['host'].lower():
                            counter +=1
                            break;
            result.append(
                (distinct_host.lower(), round((counter / self.keywords_count)*100, 2))
            )
        return result

    def get_distinct_hosts_statistic_top10(self, ):
        result = []
        for distinct_host in self.get_distinct_hosts():
            counter = 0
            for task_resaults in self.tasks_resaults:
                for keyword in task_resaults['keywords']:
                    for resault in keyword['serp'][:10]:
                        if distinct_host.lower() in resault['host'].lower():
                            counter +=1
                            break;
            result.append(
                (distinct_host.lower(), round((counter / self.keywords_count)*100, 2))
            )
        return result

    def get_task_resault(self, taskId):
        response = self.session.get(self.__BASE_API_URL+f'/task/result/{taskId}', params = {'key':self.__API_KEY})
        if response.status_code == 200:
            result = json.loads(response.text)
            self.__tasks_resaults.append(result)
            return result
        else:
            raise ConnectionError(f'Не удалось получить результаты {taskId}')


"""Spotitude.

Get a user's top 25 tracks and return a 5x5 grid visualization.
"""

from config import Config
from data import get_top_tracks
import spotipy
from visualization import make_visualization
import argparse
import gui

if __name__ == "__main__":
    # Parse the input parameters.
    parser = argparse.ArgumentParser(description="Spotitude Customization")
    parser.add_argument(
        "--time_range",
        type=str,
        default="short_term",
        help="Over what time frame the top tracks are computed. "
        + "Valid values: long_term, medium_term and short_term. (default: short_term)",
    )
    args = parser.parse_args()
    if args.time_range not in ["short_term", "medium_term", "long_term"]:
        raise ValueError(
            "Valid values for time_range are: long_term, medium_term and short_term."
        )

    # Get authentication token for Spotify API
    config = Config()
    token = config.get_token()

    if token:
        spotify = spotipy.Spotify(auth=token)
        # Get user's top tracks
        top_tracks_df = get_top_tracks(
            spotify, limit=25, time_range=args.time_range, save=True
        )

        # Create visualization
        make_visualization(top_tracks_df)

        # Open visualization using Eel as interactive GUI
        gui.open_visualization(spotify, top_tracks_df, args.time_range)


# -*- coding: utf-8 -*-
from django.contrib.auth.forms import AuthenticationForm
from django.contrib.auth.views import LoginView
from django.shortcuts import render, redirect
from .models import Contact
from .forms import ContactForm

template = 'core/'

class CustomAuthenticationForm(AuthenticationForm):
    """Definition of a Django-authentication-form-based form.

    Inherits from AuthenticationForm (class), a base class for
    authenticating users.

    Attributes
    ----------
    error_messages : dict
        Error message key-value pair.
    """
    error_messages = {
        'invalid_login': "Dados inválidos. Tente novamente."
    }

class CustomLoginView(LoginView):
    """Definition of a Django-login-view-based class.

    Inherits from LoginView (class), display the login form and
    handle the login action.

    Attributes
    ----------
    template_name : str
        The login view file path.
    authentication_form : class
        Django-authentication-form-based form.
    """
    template_name = 'accounts/login.html'
    authentication_form = CustomAuthenticationForm

def index(request):
    """Definition of the app index view.

    Parameters
    ----------
    request : HttpRequest
        The client request to the server.

    Returns
    -------
    HttpResponse
        The server response to the client.
    """
    form = ContactForm(request.POST or None)
    context = {}
    if form.is_valid():
        form.save()
        request.session['success'] = True
        request.session['was_reloaded'] = False
        return redirect('/#contact')
    else:
        if 'was_reloaded' in request.session:
            if not request.session['was_reloaded']:
                request.session['was_reloaded'] = True
            else:
                request.session['success'] = False
    context['form'] = form
    return render(request, template + 'index.html', context)

def error_500_view(request):
    """Definition of the 500 error view.

    Parameters
    ----------
    request : HttpRequest
        The client request to the server.

    Returns
    -------
    HttpResponse
        The server response to the client.
    """
    return render(request, 'core/500.html')


import pickle
import PIL
from PIL import Image
import numpy as np
import os

# for letter in ['o','p','q','r','s','u']:
for letter in ['a','b','c','e','g']:
	samples = len([f for f in os.listdir('./Letters/'+letter+'/') if f[-5:] == ".jpeg"])
	databasename='./Letters/'+letter+'/'+letter+'_vector.p'
	database=[]
	for i in range(0,samples):
		imagename='./Letters/'+letter+'/img_'+letter+'_'+str(i)
		img = Image.open(imagename+'.jpeg')
		#To create vector:
		img = img.convert('L')
		dat=np.array(img)/255
		database.append(dat)
	pickle.dump(database,open(databasename,'wb'))
	print("Exported vector for letter " + letter)


from django.conf.urls import patterns, include, url
from django.contrib import admin

urlpatterns = patterns('',
    # Examples:
    url(r'^$', 'agilecar.views.index', name='index'),
    # url(r'^blog/', include('blog.urls')),

    url(r'^admin/', include(admin.site.urls)),
    url(r'^jobposting/', include('jobposting.urls')),
    url(r'^application/', include('application.urls')),
    url(r'^screeners/', include('screener.urls')),

    #auth urls
    url(r'^account/login/$', 'agilecar.views.login'),
    url(r'^account/register/$', 'agilecar.views.register'),
    url(r'^account/logout/$', 'agilecar.views.logout'),
    #url(r'^account/loggedin/$', 'agilecar.views.loggedin'),
    #url(r'^account/invalid/$', 'agilecar.views.invalid_login'),
)


import config
import hashlib
import app

class Edit:

    def __init__(self):
        pass

    '''
    def GET(self, id_denuncia, **k):
        if app.session.loggedin is True: # validate if the user is logged
            # session_username = app.session.username
            session_privilege = app.session.privilege # get the session_privilege
            if session_privilege == 0: # admin user
                return self.GET_EDIT(id_denuncia) # call GET_EDIT function
            elif session_privilege == 1: # guess user
                raise config.web.seeother('/guess') # render guess.html
        else: # the user dont have logged
            raise config.web.seeother('/login') # render login.html

    def POST(self, id_denuncia, **k):
        if app.session.loggedin is True: # validate if the user is logged
            # session_username = app.session.username
            session_privilege = app.session.privilege # get the session_privilege
            if session_privilege == 0: # admin user
                return self.POST_EDIT(id_denuncia) # call POST_EDIT function
            elif session_privilege == 1: # guess user
                raise config.web.seeother('/guess') # render guess.html
        else: # the user dont have logged
            raise config.web.seeother('/login') # render login.html

    @staticmethod
    def GET_EDIT(id_denuncia, **k):

    @staticmethod
    def POST_EDIT(id_denuncia, **k):

    '''

    def GET(self, id_denuncia, **k):
        message = None # Error message
        id_denuncia = config.check_secure_val(str(id_denuncia)) # HMAC id_denuncia validate
        result = config.model.get_denuncia(int(id_denuncia)) # search for the id_denuncia
        result.id_denuncia = config.make_secure_val(str(result.id_denuncia)) # apply HMAC for id_denuncia
        return config.render.edit(result, message) # render denuncia edit.html

    def POST(self, id_denuncia, **k):
        form = config.web.input()  # get form data
        form['id_denuncia'] = config.check_secure_val(str(form['id_denuncia'])) # HMAC id_denuncia validate
        # edit user with new data
        result = config.model.edit_denuncia(
            form['id_denuncia'],form['descripcion_d'],form['calle_d'],form['municipio_d'],form['colonia_d'],form['longitud'],form['latitud'],
        )
        if result == None: # Error on udpate data
            id_denuncia = config.check_secure_val(str(id_denuncia)) # validate HMAC id_denuncia
            result = config.model.get_denuncia(int(id_denuncia)) # search for id_denuncia data
            result.id_denuncia = config.make_secure_val(str(result.id_denuncia)) # apply HMAC to id_denuncia
            message = "Error al editar el registro" # Error message
            return config.render.edit(result, message) # render edit.html again
        else: # update user data succefully
            raise config.web.seeother('/denuncia') # render denuncia index.html


import copy
import os
from collections import OrderedDict
from logging import getLogger
from time import time

import numpy as num
import theano.tensor as tt
from pymc3 import Deterministic, Uniform
from pyrocko.gf import LocalEngine, RectangularSource
from theano import config as tconfig
from theano import shared
from theano.printing import Print

from beat import config as bconfig
from beat import covariance as cov
from beat import heart, theanof, utility
from beat.ffi import get_gf_prefix, load_gf_library
from beat.interseismic import geo_backslip_synthetics, seperate_point
from beat.models.base import (
    Composite,
    ConfigInconsistentError,
    FaultGeometryNotFoundError,
    get_hypervalue_from_point,
)
from beat.models.distributions import multivariate_normal_chol

logger = getLogger("geodetic")

km = 1000.0

__all__ = [
    "GeodeticGeometryComposite",
    "GeodeticInterseismicComposite",
    "GeodeticDistributerComposite",
]

class GeodeticComposite(Composite):
    """
    Comprises data structure of the geodetic composite.

    Parameters
    ----------
    gc : :class:`config.GeodeticConfig`
        configuration object containing seismic setup parameters
    project_dir : str
        directory of the model project, where to find the data
    events : list
        of :class:`pyrocko.model.Event`
        contains information of reference event(s), coordinates of reference
        point(s) and source time(s)
    hypers : boolean
        if true initialise object for hyper parameter optimization
    """

    _hierarchicalnames = None
    weights = None

    def __init__(self, gc, project_dir, events, hypers=False):

        super(GeodeticComposite, self).__init__(events)

        logger.debug("Setting up geodetic structure ...\n")
        self.name = "geodetic"
        self._like_name = "geo_like"

        geodetic_data_path = os.path.join(project_dir, bconfig.geodetic_data_name)

        self.datasets = utility.load_objects(geodetic_data_path)
        logger.info("Number of geodetic datasets: %i " % self.n_t)

        # init geodetic targets
        self.targets = heart.init_geodetic_targets(
            datasets=self.datasets,
            earth_model_name=gc.gf_config.earth_model_name,
            interpolation=gc.interpolation,
            crust_inds=[gc.gf_config.reference_model_idx],
            sample_rate=gc.gf_config.sample_rate,
        )

        # merge geodetic data to calculate residuals on single array
        datasets, los_vectors, odws, self.Bij = heart.concatenate_datasets(
            self.datasets
        )
        logger.info("Number of geodetic data points: %i " % self.Bij.ordering.size)

        self.sdata = shared(datasets, name="geodetic_data", borrow=True)
        self.slos_vectors = shared(los_vectors, name="los_vecs", borrow=True)
        self.sodws = shared(odws, name="odws", borrow=True)

        self.noise_analyser = cov.GeodeticNoiseAnalyser(
            config=gc.noise_estimator, events=self.events
        )

        if gc.corrections_config.has_enabled_corrections:
            correction_configs = gc.corrections_config.iter_corrections()
            logger.info("Initialising corrections ...")
            for data in self.datasets:
                data.setup_corrections(
                    event=self.event, correction_configs=correction_configs
                )

        self.config = gc

        if hypers:
            self._llks = []
            for t in range(self.n_t):
                self._llks.append(
                    shared(num.array([1.0]), name="geo_llk_%i" % t, borrow=True)
                )

    def init_weights(self):
        self.weights = []
        for i, data in enumerate(self.datasets):
            if int(data.covariance.data.sum()) == data.ncoords:
                logger.warning(
                    "Data covariance is identity matrix! Please double check!!!"
                )

            choli = data.covariance.chol_inverse
            self.weights.append(shared(choli, name="geo_weight_%i" % i, borrow=True))
            data.covariance.update_slog_pdet()

    @property
    def n_t(self):
        return len(self.datasets)

    def get_all_dataset_names(self, hp_name):
        """
        Return unique GNSS stations and radar acquisitions.
        """
        names = []
        for dataset in self.datasets:
            if dataset.typ == hp_name.split("_")[1]:
                if isinstance(dataset, heart.DiffIFG):
                    names.append(dataset.name)
                elif isinstance(dataset, heart.GNSSCompoundComponent):
                    names.append(dataset.component)
                else:
                    TypeError(
                        'Geodetic Dataset of class "%s" not '
                        "supported" % dataset.__class__.__name__
                    )

        return names

    def analyse_noise(self, tpoint=None):
        """
        Analyse geodetic noise in datasets and set
        data-covariance matrixes accordingly.
        """
        if self.config.noise_estimator.structure == "non-toeplitz":
            results = self.assemble_results(tpoint)
        else:
            results = [None] * len(self.datasets)

        if len(self.datasets) != len(results):
            raise ValueError("Number of datasets and results need to be equal!")

        for dataset, result in zip(self.datasets, results):
            logger.info(
                'Retrieving geodetic data-covariances with structure "%s" '
                "for %s ..." % (self.config.noise_estimator.structure, dataset.name)
            )

            cov_d_geodetic = self.noise_analyser.get_data_covariance(
                dataset, result=result
            )

            if dataset.covariance is None:
                dataset.covariance = heart.Covariance(data=cov_d_geodetic)
            else:
                dataset.covariance.data = cov_d_geodetic

            if int(dataset.covariance.data.sum()) == dataset.ncoords:
                logger.warning(
                    "Data covariance is identity matrix! Please double check!!!"
                )

    def get_hypersize(self, hp_name=""):
        """
        Return size of the hyperparameter

        Parameters
        ----------
        hp_name: str
            of hyperparameter name

        Returns
        -------
        int
        """
        n_datasets = len(self.get_all_dataset_names(hp_name))
        if n_datasets == 0:
            raise ConfigInconsistentError(
                'Found no data for hyperparameter "%s". Please either load'
                " the data or remove it from types dictionary!" % hp_name,
                params="hypers",
            )
        elif self.config.dataset_specific_residual_noise_estimation:
            return n_datasets
        else:
            return 1

    def assemble_results(self, point, **kwargs):
        """
        Assemble geodetic data for given point in solution space.

        Parameters
        ----------
        point : :func:`pymc3.Point`
            Dictionary with model parameters

        Returns
        -------
        List with :class:`heart.GeodeticResult`
        """

        logger.debug("Assembling geodetic data ...")

        processed_synts = self.get_synthetics(point, outmode="stacked_arrays")

        results = []
        for i, data in enumerate(self.datasets):
            res = data.displacement - processed_synts[i]

            results.append(
                heart.GeodeticResult(
                    point=point,
                    processed_obs=data.displacement,
                    processed_syn=processed_synts[i],
                    processed_res=res,
                )
            )

        return results

    def export(
        self,
        point,
        results_path,
        stage_number,
        fix_output=False,
        force=False,
        update=False,
    ):

        from kite.scene import Scene, UserIOWarning
        from pyrocko.guts import dump

        from beat.plotting import map_displacement_grid

        def save_covs(datasets, cov_mat="pred_v"):
            """
            Save covariance matrixes of given attribute
            """

            covs = {
                dataset.name: getattr(dataset.covariance, cov_mat)
                for dataset in datasets
            }

            outname = os.path.join(results_path, "%s_C_%s" % ("geodetic", cov_mat))
            logger.info('"geodetic covariances" to: %s', outname)
            num.savez(outname, **covs)

        gc = self.config

        results = self.assemble_results(point)

        def get_filename(attr, ending="csv"):
            return os.path.join(
                results_path,
                "{}_{}_{}.{}".format(
                    os.path.splitext(dataset.name)[0], attr, stage_number, ending
                ),
            )

        # export for gnss
        for typ, config in gc.types.items():
            if "GNSS" == typ:
                from pyrocko.model import gnss

                logger.info("Exporting GNSS data ...")
                campaigns = config.load_data(campaign=True)

                for campaign in campaigns:
                    model_camp = gnss.GNSSCampaign(
                        stations=copy.deepcopy(campaign.stations),
                        name="%s_model" % campaign.name,
                    )

                    dataset_to_result = {}
                    for dataset, result in zip(self.datasets, results):
                        if dataset.typ == "GNSS":
                            dataset_to_result[dataset] = result

                    for dataset, result in dataset_to_result.items():
                        for ista, sta in enumerate(model_camp.stations):
                            comp = getattr(sta, dataset.component)
                            comp.shift = result.processed_syn[ista]
                            comp.sigma = 0.0

                    outname = os.path.join(
                        results_path, "gnss_synths_%i.yaml" % stage_number
                    )

                    dump(model_camp, filename=outname)

            elif "SAR" == typ:
                logger.info("Exporting SAR data ...")
                for dataset, result in zip(self.datasets, results):
                    if dataset.typ == "SAR":
                        try:
                            scene_path = os.path.join(config.datadir, dataset.name)
                            logger.info(
                                "Loading full resolution kite scene: %s" % scene_path
                            )
                            scene = Scene.load(scene_path)
                        except UserIOWarning:
                            logger.warning(
                                "Full resolution data could not be"
                                " loaded! Skipping ..."
                            )
                            continue

                        for attr in ["processed_obs", "processed_syn", "processed_res"]:

                            filename = get_filename(attr, ending="csv")
                            displacements = getattr(result, attr)
                            dataset.export_to_csv(filename, displacements)
                            logger.info("Stored CSV file to: %s" % filename)

                            filename = get_filename(attr, ending="yml")
                            vals = map_displacement_grid(displacements, scene)
                            scene.displacement = vals
                            scene.save(filename)
                            logger.info("Stored kite scene to: %s" % filename)

        # export stdz residuals
        self.analyse_noise(point)
        if update:
            logger.info("Saving velocity model covariance matrixes...")
            self.update_weights(point)
            save_covs(self.datasets, "pred_v")

        logger.info("Saving data covariance matrixes...")
        save_covs(self.datasets, "data")

    def init_hierarchicals(self, problem_config):
        """
        Initialize hierarchical parameters.
        Ramp estimation in azimuth and range direction of a radar scene and/or
        Rotation of GNSS stations around an Euler pole
        """
        hierarchicals = problem_config.hierarchicals
        self._hierarchicalnames = []
        for number, corr in enumerate(
            self.config.corrections_config.iter_corrections()
        ):
            logger.info(
                "Evaluating config for %s corrections " "for datasets..." % corr.feature
            )
            if corr.enabled:
                for data in self.datasets:
                    if data.name in corr.dataset_names:
                        hierarchical_names = corr.get_hierarchical_names(
                            name=data.name, number=number
                        )
                    else:
                        hierarchical_names = []

                    for hierarchical_name in hierarchical_names:
                        if not corr.enabled and hierarchical_name in hierarchicals:

                            raise ConfigInconsistentError(
                                "%s %s disabled, but they are defined"
                                " in the problem configuration"
                                " (hierarchicals)!" % (corr.feature, data.name)
                            )

                        if (
                            corr.enabled
                            and hierarchical_name not in hierarchicals
                            and data.name in corr.dataset_names
                        ):
                            raise ConfigInconsistentError(
                                "%s %s corrections enabled, but they are"
                                " not defined in the problem configuration!"
                                " (hierarchicals)" % (corr.feature, data.name)
                            )

                        param = hierarchicals[hierarchical_name]
                        if hierarchical_name not in self.hierarchicals:
                            if not num.array_equal(param.lower, param.upper):
                                kwargs = dict(
                                    name=param.name,
                                    shape=param.dimension,
                                    lower=param.lower,
                                    upper=param.upper,
                                    testval=param.testvalue,
                                    transform=None,
                                    dtype=tconfig.floatX,
                                )

                                try:
                                    self.hierarchicals[hierarchical_name] = Uniform(
                                        **kwargs
                                    )
                                except TypeError:
                                    kwargs.pop("name")
                                    self.hierarchicals[
                                        hierarchical_name
                                    ] = Uniform.dist(**kwargs)

                                self._hierarchicalnames.append(hierarchical_name)
                            else:
                                logger.info(
                                    "not solving for %s, got fixed at %s"
                                    % (
                                        param.name,
                                        utility.list2string(param.lower.flatten()),
                                    )
                                )
                                self.hierarchicals[hierarchical_name] = param.lower
            else:
                logger.info("No %s correction!" % corr.feature)

        logger.info("Initialized %i hierarchical parameters." % len(self.hierarchicals))

    def apply_corrections(self, residuals, point=None, operation="-"):
        """
        Apply all the configured correction terms e.g. SAR orbital ramps,
        GNSS Euler pole rotations etc...
        """
        for i, dataset in enumerate(self.datasets):
            if dataset.has_correction:
                for corr in dataset.corrections:
                    correction = corr.get_displacements(self.hierarchicals, point=point)
                    # correction = Print('corr')(correction)

                    if operation == "-":
                        residuals[i] -= correction  # needs explicit assignment!
                    elif operation == "+":
                        residuals[i] += correction

        return residuals

    def update_llks(self, point):
        """
        Update posterior likelihoods (in place) of the composite w.r.t.
        one point in the solution space.

        Parameters
        ----------
        point : dict
            with numpy array-like items and variable name keys
        """
        results = self.assemble_results(point)
        for l, result in enumerate(results):
            choli = self.datasets[l].covariance.chol_inverse
            tmp = choli.dot(result.processed_res)
            _llk = num.asarray([num.dot(tmp, tmp)])
            self._llks[l].set_value(_llk)

    def get_variance_reductions(self, point, results=None, weights=None):
        """
        Parameters
        ----------
        point : dict
            with parameters to point in solution space to calculate
            variance reductions

        Returns
        -------
        dict of floats,
            keys are nslc_ids
        """
        if results is None:
            results = self.assemble_results(point)

        ndatasets = len(self.datasets)

        assert len(results) == ndatasets

        if weights is None:
            self.analyse_noise(point)
            self.update_weights(point)
            weights = self.weights

        nweights = len(weights)
        assert nweights == ndatasets

        logger.debug("n weights %i , n datasets %i" % (nweights, ndatasets))

        assert nweights == ndatasets

        logger.debug("Calculating variance reduction for solution ...")

        counter = utility.Counter()
        hp_specific = self.config.dataset_specific_residual_noise_estimation

        var_reds = OrderedDict()
        for dataset, weight, result in zip(self.datasets, weights, results):

            hp = get_hypervalue_from_point(
                point, dataset, counter, hp_specific=hp_specific
            )
            icov = dataset.covariance.inverse(num.exp(hp * 2.0))

            data = result.processed_obs
            residual = result.processed_res

            nom = residual.T.dot(icov).dot(residual)
            denom = data.T.dot(icov).dot(data)

            logger.debug("nom %f, denom %f" % (float(nom), float(denom)))
            var_red = 1 - (nom / denom)

            logger.debug("Variance reduction for %s is %f" % (dataset.name, var_red))

            if 0:
                from matplotlib import pyplot as plt

                fig, ax = plt.subplots(1, 1)
                im = ax.imshow(dataset.covariance.data)
                plt.colorbar(im)
                plt.show()

            var_reds[dataset.name] = var_red

        return var_reds

    def get_standardized_residuals(self, point, results=None, weights=None):
        """
        Parameters
        ----------
        point : dict
            with parameters to point in solution space to calculate
            standardized residuals

        Returns
        -------
        dict of arrays of standardized residuals,
            keys are nslc_ids
        """
        if results is None:
            results = self.assemble_results(point)

        if weights is None:
            self.update_weights(point)

        counter = utility.Counter()
        hp_specific = self.config.dataset_specific_residual_noise_estimation

        stdz_residuals = OrderedDict()
        for dataset, result in zip(self.datasets, results):
            hp = get_hypervalue_from_point(
                point, dataset, counter, hp_specific=hp_specific
            )
            choli = num.linalg.inv(dataset.covariance.chol(num.exp(hp * 2.0)))
            stdz_residuals[dataset.name] = choli.dot(result.processed_res)

        return stdz_residuals

class GeodeticSourceComposite(GeodeticComposite):
    """
    Comprises how to solve the non-linear geodetic forward model.

    Parameters
    ----------
    gc : :class:`config.GeodeticConfig`
        configuration object containing seismic setup parameters
    project_dir : str
        directory of the model project, where to find the data
    sources : list
        of :class:`pyrocko.gf.seismosizer.Source`
    events : list
        of :class:`pyrocko.model.Event`
        contains information of reference event, coordinates of reference
        point and source time
    hypers : boolean
        if true initialise object for hyper parameter optimization
    """

    def __init__(self, gc, project_dir, sources, events, hypers=False):

        super(GeodeticSourceComposite, self).__init__(
            gc, project_dir, events, hypers=hypers
        )

        self.engine = LocalEngine(store_superdirs=[gc.gf_config.store_superdir])

        self.sources = sources

    def __getstate__(self):
        self.engine.close_cashed_stores()
        return self.__dict__.copy()

    def point2sources(self, point):
        """
        Updates the composite source(s) (in place) with the point values.
        """
        tpoint = copy.deepcopy(point)
        tpoint.update(self.fixed_rvs)
        tpoint = utility.adjust_point_units(tpoint)

        # remove hyperparameters from point
        hps = self.config.get_hypernames()

        for hyper in hps:
            if hyper in tpoint:
                tpoint.pop(hyper)

        source_params = list(self.sources[0].keys())
        for param in list(tpoint.keys()):
            if param not in source_params:
                tpoint.pop(param)

        source_points = utility.split_point(tpoint)
        for i, source in enumerate(self.sources):
            utility.update_source(source, **source_points[i])
            # reset source time may result in store error otherwise
            source.time = 0.0

    def get_pyrocko_events(self, point=None):
        """
        Transform sources to pyrocko events.

        Returns
        -------
        events : list
            of :class:`pyrocko.model.Event`
        """

        if point is not None:
            self.point2sources(point)

        events = []
        target = self.targets[0]
        store = self.engine.get_store(target.store_id)
        for source in self.sources:
            events.append(source.pyrocko_event(store=store, target=target))

        return events

    def get_formula(self, input_rvs, fixed_rvs, hyperparams, problem_config):
        """
        Get geodetic likelihood formula for the model built. Has to be called
        within a with model context.
        Part of the pymc3 model.

        Parameters
        ----------
        input_rvs : dict
            of :class:`pymc3.distribution.Distribution`
        fixed_rvs : dict
            of :class:`numpy.array`
        hyperparams : dict
            of :class:`pymc3.distribution.Distribution`
        problem_config : :class:`config.ProblemConfig`

        Returns
        -------
        posterior_llk : :class:`theano.tensor.Tensor`
        """
        hp_specific = self.config.dataset_specific_residual_noise_estimation
        tpoint = problem_config.get_test_point()

        self.input_rvs = input_rvs
        self.fixed_rvs = fixed_rvs

        logger.info(
            "Geodetic optimization on: \n " "%s" % ", ".join(self.input_rvs.keys())
        )

        self.input_rvs.update(fixed_rvs)

        t0 = time()
        disp = self.get_synths(self.input_rvs)
        t1 = time()
        logger.debug("Geodetic forward model on test model takes: %f" % (t1 - t0))

        los_disp = (disp * self.slos_vectors).sum(axis=1)

        residuals = self.Bij.srmap(
            tt.cast((self.sdata - los_disp) * self.sodws, tconfig.floatX)
        )

        self.init_hierarchicals(problem_config)
        self.analyse_noise(tpoint)
        self.init_weights()
        if self.config.corrections_config.has_enabled_corrections:
            logger.info("Applying corrections! ...")
            residuals = self.apply_corrections(residuals, operation="-")

        logpts = multivariate_normal_chol(
            self.datasets, self.weights, hyperparams, residuals, hp_specific=hp_specific
        )

        llk = Deterministic(self._like_name, logpts)
        return llk.sum()

class GeodeticGeometryComposite(GeodeticSourceComposite):
    def __init__(self, gc, project_dir, sources, events, hypers=False):

        super(GeodeticGeometryComposite, self).__init__(
            gc, project_dir, sources, events, hypers=hypers
        )

        if not hypers:
            # synthetics generation
            logger.debug("Initialising synthetics functions ... \n")
            self.get_synths = theanof.GeoSynthesizer(
                engine=self.engine, sources=self.sources, targets=self.targets
            )

    def get_synthetics(self, point, **kwargs):
        """
        Get synthetics for given point in solution space.

        Parameters
        ----------
        point : :func:`pymc3.Point`
            Dictionary with model parameters
        kwargs especially to change output of the forward model

        Returns
        -------
        list with :class:`numpy.ndarray` synthetics for each target
        """
        self.point2sources(point)

        displacements = heart.geo_synthetics(
            engine=self.engine, targets=self.targets, sources=self.sources, **kwargs
        )

        synths = []
        for disp, data in zip(displacements, self.datasets):
            los_d = (disp * data.los_vector).sum(axis=1)
            synths.append(los_d)

        if self.config.corrections_config.has_enabled_corrections:
            synths = self.apply_corrections(synths, point=point, operation="+")

        return synths

    def update_weights(self, point, n_jobs=1, plot=False):
        """
        Updates weighting matrixes (in place) with respect to the point in the
        solution space.

        Parameters
        ----------
        point : dict
            with numpy array-like items and variable name keys
        """
        gc = self.config

        if not self.weights:
            self.init_weights()

        self.point2sources(point)

        # update data covariances in case model dependent non-toeplitz
        if self.config.noise_estimator.structure == "non-toeplitz":
            logger.info("Updating data-covariances ...")
            self.analyse_noise(point)

        crust_inds = range(*gc.gf_config.n_variations)
        thresh = 5
        if len(crust_inds) > thresh:
            logger.info("Updating geodetic velocity model-covariances ...")
            if self.config.noise_estimator.structure == "non-toeplitz":
                logger.warning(
                    "Non-toeplitz estimation in combination with model "
                    "prediction covariances is still EXPERIMENTAL and results"
                    " should be interpreted with care!!"
                )

            for i, data in enumerate(self.datasets):
                crust_targets = heart.init_geodetic_targets(
                    datasets=[data],
                    earth_model_name=gc.gf_config.earth_model_name,
                    interpolation=gc.interpolation,
                    crust_inds=crust_inds,
                    sample_rate=gc.gf_config.sample_rate,
                )

                logger.debug("Track %s" % data.name)
                cov_pv = cov.geodetic_cov_velocity_models(
                    engine=self.engine,
                    sources=self.sources,
                    targets=crust_targets,
                    dataset=data,
                    plot=plot,
                    event=self.event,
                    n_jobs=1,
                )

                cov_pv = utility.ensure_cov_psd(cov_pv)
                data.covariance.pred_v = cov_pv
                choli = data.covariance.chol_inverse

                self.weights[i].set_value(choli)
                data.covariance.update_slog_pdet()
        else:
            logger.info(
                "Not updating geodetic velocity model-covariances because "
                "number of model variations is too low! < %i" % thresh
            )

class GeodeticInterseismicComposite(GeodeticSourceComposite):
    def __init__(self, gc, project_dir, sources, events, hypers=False):

        super(GeodeticInterseismicComposite, self).__init__(
            gc, project_dir, sources, events, hypers=hypers
        )

        for source in sources:
            if not isinstance(source, RectangularSource):
                raise TypeError("Sources have to be RectangularSources!")

        if not hypers:
            self._lats = self.Bij.l2a([data.lats for data in self.datasets])
            self._lons = self.Bij.l2a([data.lons for data in self.datasets])

            self.get_synths = theanof.GeoInterseismicSynthesizer(
                lats=self._lats,
                lons=self._lons,
                engine=self.engine,
                targets=self.targets,
                sources=sources,
                reference=self.event,
            )

    def get_synthetics(self, point, **kwargs):
        """
        Get synthetics for given point in solution space.

        Parameters
        ----------
        point : :func:`pymc3.Point`
            Dictionary with model parameters
        kwargs especially to change output of the forward model

        Returns
        -------
        list with :class:`numpy.ndarray` synthetics for each target
        """
        tpoint = copy.deepcopy(point)
        tpoint.update(self.fixed_rvs)
        spoint, bpoint = seperate_point(tpoint)

        self.point2sources(spoint)

        synths = []
        for target, data in zip(self.targets, self.datasets):
            disp = geo_backslip_synthetics(
                engine=self.engine,
                sources=self.sources,
                targets=[target],
                lons=target.lons,
                lats=target.lats,
                reference=self.event,
                **bpoint
            )
            synths.append((disp * data.los_vector).sum(axis=1))

        return synths

    def update_weights(self, point, n_jobs=1, plot=False):

        if not self.weights:
            self.init_weights()

        logger.warning("Not implemented yet!")
        raise NotImplementedError("Not implemented yet!")

class GeodeticDistributerComposite(GeodeticComposite):
    """
    Comprises how to solve the geodetic (static) linear forward model.
    Distributed slip
    """

    def __init__(self, gc, project_dir, events, hypers=False):

        super(GeodeticDistributerComposite, self).__init__(
            gc, project_dir, events, hypers=hypers
        )

        self.gfs = {}
        self.gf_names = {}

        self._mode = "ffi"
        self.gfpath = os.path.join(project_dir, self._mode, bconfig.linear_gf_dir_name)

        self.fault = None

    def get_gflibrary_key(self, crust_ind, wavename, component):
        return "%i_%s_%s" % (crust_ind, wavename, component)

    def load_gfs(self, crust_inds=None, make_shared=True):
        """
        Load Greens Function matrixes for each variable to be inverted for.
        Updates gfs and gf_names attributes.

        Parameters
        ----------
        crust_inds : list
            of int to indexes of Green's Functions
        make_shared : bool
            if True transforms gfs to :class:`theano.shared` variables
        """

        if crust_inds is None:
            crust_inds = range(*self.config.gf_config.n_variations)

        if not isinstance(crust_inds, list):
            raise TypeError("crust_inds need to be a list!")

        for crust_ind in crust_inds:
            gfs = {}
            for var in self.slip_varnames:
                gflib_name = get_gf_prefix(
                    datatype=self.name,
                    component=var,
                    wavename="static",
                    crust_ind=crust_ind,
                )
                gfpath = os.path.join(self.gfpath, gflib_name)

                gfs = load_gf_library(directory=self.gfpath, filename=gflib_name)

                if make_shared:
                    gfs.init_optimization()

                key = self.get_gflibrary_key(
                    crust_ind=crust_ind, wavename="static", component=var
                )

                self.gf_names[key] = gfpath
                self.gfs[key] = gfs

    def load_fault_geometry(self):
        """
        Load fault-geometry, i.e. discretized patches.

        Returns
        -------
        :class:`heart.FaultGeometry`
        """
        try:
            return utility.load_objects(
                os.path.join(self.gfpath, bconfig.fault_geometry_name)
            )[0]
        except Exception:
            raise FaultGeometryNotFoundError()

    def point2sources(self, point):
        """
        Returns the fault source patche(s) with the point values updated.

        Parameters
        ----------
        point : dict
            with random variables from solution space
        """
        tpoint = copy.deepcopy(point)
        # tpoint.update(self.fixed_rvs)   if vars are fixed GFS are not calculated

        if self.nevents == 1:
            events = [self.event]  # single event
        else:
            events = self.events  # multi event

        if self.fault is None:
            self.fault = self.load_fault_geometry()

        return self.fault.point2sources(tpoint, events=events)

    def get_formula(self, input_rvs, fixed_rvs, hyperparams, problem_config):
        """
        Formulation of the distribution problem for the model built. Has to be
        called within a with-model-context.

        Parameters
        ----------
        input_rvs : list
            of :class:`pymc3.distribution.Distribution`
        hyperparams : dict
            of :class:`pymc3.distribution.Distribution`

        Returns
        -------
        llk : :class:`theano.tensor.Tensor`
            log-likelihood for the distributed slip
        """
        logger.info("Loading %s Green's Functions" % self.name)
        self.load_gfs(
            crust_inds=[self.config.gf_config.reference_model_idx], make_shared=False
        )

        tpoint = problem_config.get_test_point()
        self.analyse_noise(tpoint)
        for gfs in self.gfs.values():
            gfs.init_optimization()

        self.init_weights()

        hp_specific = self.config.dataset_specific_residual_noise_estimation

        self.input_rvs = input_rvs
        self.fixed_rvs = fixed_rvs
        ref_idx = self.config.gf_config.reference_model_idx

        mu = tt.zeros((self.Bij.ordering.size), tconfig.floatX)
        for var in self.slip_varnames:
            key = self.get_gflibrary_key(
                crust_ind=ref_idx, wavename="static", component=var
            )
            mu += self.gfs[key].stack_all(slips=input_rvs[var])

        residuals = self.Bij.srmap(
            tt.cast((self.sdata - mu) * self.sodws, tconfig.floatX)
        )

        self.init_hierarchicals(problem_config)
        if self.config.corrections_config.has_enabled_corrections:
            residuals = self.apply_corrections(residuals)

        logpts = multivariate_normal_chol(
            self.datasets, self.weights, hyperparams, residuals, hp_specific=hp_specific
        )

        llk = Deterministic(self._like_name, logpts)

        return llk.sum()

    def get_synthetics(self, point, outmode="data"):
        """
        Get synthetics for given point in solution space.

        Parameters
        ----------
        point : :func:`pymc3.Point`
            Dictionary with model parameters
        kwargs especially to change output of the forward model

        Returns
        -------
        list with :class:`numpy.ndarray` synthetics for each target
        """

        ref_idx = self.config.gf_config.reference_model_idx
        if len(self.gfs) == 0:
            self.load_gfs(crust_inds=[ref_idx], make_shared=False)

        for gfs in self.gfs.values():
            gfs.set_stack_mode("numpy")

        tpoint = copy.deepcopy(point)

        hps = self.config.get_hypernames()

        for hyper in hps:
            if hyper in tpoint:
                tpoint.pop(hyper)

        mu = num.zeros((self.Bij.ordering.size))
        for var in self.slip_varnames:
            key = self.get_gflibrary_key(
                crust_ind=ref_idx, wavename="static", component=var
            )
            mu += self.gfs[key].stack_all(slips=point[var])

        synths = self.Bij.a2l(mu)

        if self.config.corrections_config.has_enabled_corrections:
            synths = self.apply_corrections(synths, point=point, operation="+")

        return synths

    def update_weights(self, point, n_jobs=1, plot=False):
        """
        Updates weighting matrixes (in place) with respect to the point in the
        solution space.

        Parameters
        ----------
        point : dict
            with numpy array-like items and variable name keys
        """
        gc = self.config

        if not self.weights:
            self.init_weights()

        # update data covariances in case model dependent non-toeplitz
        if self.config.noise_estimator.structure == "non-toeplitz":
            logger.info("Updating data-covariances ...")
            self.analyse_noise(point)

        crust_inds = range(*gc.gf_config.n_variations)
        thresh = 5
        if len(crust_inds) > thresh:
            logger.info("Updating geodetic velocity model-covariances ...")
            if self.config.noise_estimator.structure == "non-toeplitz":
                logger.warning(
                    "Non-toeplitz estimation in combination with model "
                    "prediction covariances is still EXPERIMENTAL and results"
                    " should be interpreted with care!!"
                )

            crust_inds = list(range(*self.config.gf_config.n_variations))
            n_variations = len(crust_inds)
            if len(self.gfs) != n_variations:
                logger.info("Loading geodetic linear GF matrixes ...")
                self.load_gfs(crust_inds=crust_inds, make_shared=False)

            crust_displacements = num.zeros((n_variations, self.Bij.ordering.size))
            for i, crust_ind in enumerate(crust_inds):
                mu = num.zeros((self.Bij.ordering.size))
                for var in self.slip_varnames:
                    key = self.get_gflibrary_key(
                        crust_ind=crust_ind, wavename="static", component=var
                    )
                    mu += self.gfs[key].stack_all(slips=point[var])

                crust_displacements[i, :] = mu

            crust_synths = self.Bij.a_nd2l(crust_displacements)
            if len(crust_synths) != self.n_t:
                raise ValueError(
                    "Number of datasets %i and number of synthetics %i "
                    "inconsistent!" % (self.n_t, len(crust_synths))
                )

            for i, data in enumerate(self.datasets):
                logger.debug("Track %s" % data.name)
                cov_pv = num.cov(crust_synths[i], rowvar=0)

                cov_pv = utility.ensure_cov_psd(cov_pv)
                data.covariance.pred_v = cov_pv
                choli = data.covariance.chol_inverse

                self.weights[i].set_value(choli)
                data.covariance.update_slog_pdet()
        else:
            logger.info(
                "Not updating geodetic velocity model-covariances because "
                "number of model variations is too low! < %i" % thresh
            )


from typing import Any, Union

from prometheus_client import Counter, Histogram

from metrics.event import Event

class SampleEvent(Event):
    LABELS = ["error"]
    METRICS = {
        "result": Counter("sample_result", "sample result", LABELS),
        "latency": Histogram(
            "sample_latency",
            "sample latency",
            LABELS,
            buckets=[1000, 2000, 3000, 5000, 10000],  # 1s to 10s
        ),
    }

    # protected

    def _transform_result_to_labels(self, result: Union[dict, Any]) -> dict:
        if isinstance(result, dict) and "error" in result:
            return {"error": result["error"]}
        else:
            return {"error": None}


#!/usr/bin/python

import sys
import argparse
from netaddr import *

def main(args):
	parser = argparse.ArgumentParser(description='Check if ip(s) in range(s) --- don\'t use -i and -I or -r and -R')
	igroup = parser.add_mutually_exclusive_group(required = True)
	igroup.add_argument('-i', '--ip',  help='ip to test')
	igroup.add_argument('-I', '--ipFile',  help='txt of ips, 1 per line')
	rgroup = parser.add_mutually_exclusive_group(required = True)
	rgroup.add_argument('-r', '--range',  help='range to check')
	rgroup.add_argument('-R', '--rangeFile',  help='file of ranges, csv with 2nd column name of range')
	parser.add_argument('-o', '--outputfile', help='name of file to output, otherwise stdout')

	args = parser.parse_args()

	rangeName = {}

	if args.ip:
		fips = [args.ip]
		try:
			IPAddress(args.ip)
		except:
			print("Not a valid ip address")
			sys.exit(1)

	if args.ipFile:
		try:
			fips = open(args.ipFile, 'r')
		except:
			print(args.ipFile + " does not exist. Please use full path if error persists")
			sys.exit(1)

	if args.range:
		try:
			range = IPNetwork(args.range)
			rangeIPs[range] = list(range)
			rangeName[range] = "N/A"
		except:
			print("Not a valid network range")
			sys.exit(1)

	count = 0
	if args.rangeFile:
		try:
			frange = open(args.rangeFile, 'r')
		except:
			print(args.rangeFile + " does not exist. Please use full path if error persists")
			sys.exit(1)
		print("Processing IP Ranges\n")
		for line in frange:
			line = line.split(",")
			try:
				range = IPNetwork(line[0])
				if len(line) > 1:
					name = line[1]
				else:
					name = "N/A"
				rangeName[range] = name.strip()
			except:
				print(line[0] + " is not a valid CIDR range.  Will not be included in search")
				ans = "Y"
				while ans != "y" and ans != "n":
					ans = input("Would you like to continue anyway? (Y/N) ")
					ans = ans.lower()
				if ans == "n":
					sys.exit(1)
			count = count + 1
			if count % 25 == 0:
				print("Processed " + str(count) + " ranges\n")

	pflag = True
	if args.outputfile:
		fout = open(args.outputfile, 'w+')
		pflag = False

	count = 0
	flag = False
	print ("Checking IPs\n")
	for ip in fips:
		flag = False
		try:
			ip = IPAddress(ip)
			for range in rangeName:
				if ip >= range[0] and ip <= range[-1]:
					s = str(ip) + "," + str(range) + "," + rangeName[range]
					if pflag:
						print(s)
					else:
						fout.write(s + "\n")
					flag = True
					break
			if flag == False:
				s = str(ip) + " not in ranges"
				if pflag:
					print(s)
				else:
					fout.write(s + "\n")
		except:
			print(ip.strip() + " is not a valid IP address.  Will not be included in search")
		count = count + 1
		if count % 100 == 0:
			print("Processed " + str(count) + " IPs\n")

if __name__ == "__main__":
	if sys.version_info < (3,0):
		print("Must use python 3.0 or +")
		sys.exit(1)
	print("netaddr module must be installed for this script to run correctly!\n")
	main(sys.argv)


"""tox-pyenv

Plugin for the tox_get_python_executable using tox's plugin system:

    https://testrun.org/tox/latest/plugins.html#tox.hookspecs.tox_get_python_executable

Modified to instead use `pyenv which` to locate the
appropriate python executable. This takes the place
of the standard behavior in tox. The built-in default
for the tox_get_python_exeucutable function
is the following (for sys.platform != 'win32'):

    @hookimpl
    def tox_get_python_executable(envconfig):
        return py.path.local.sysfind(envconfig.basepython)

which uses the 'py' package's sysfind():

    https://pylib.readthedocs.org/en/latest/path.html#py._path.local.LocalPath.sysfind

If `pyenv`'s shims are not at the very front of your path,
sysfind might lookup the global system version of python
instead of preferring a version specified by using `pyenv local`
or `pyenv global`. This plugin changes the way tox finds
your python executable to exclusively use `pyenv which`.

    https://github.com/yyuu/pyenv/blob/master/COMMANDS.md#pyenv-which

"""

# __about__
__title__ = 'tox-pyenv'
__summary__ = ('tox plugin that makes tox use `pyenv which` '
               'to find python executables')
__url__ = 'https://github.com/samstav/tox-pyenv'
__version__ = '1.1.0'
__author__ = 'Sam Stavinoha'
__email__ = 'smlstvnh@gmail.com'
__keywords__ = ['tox', 'pyenv', 'python']
__license__ = 'Apache License, Version 2.0'
# __about__

import logging
import subprocess

import py
from tox import hookimpl as tox_hookimpl

LOG = logging.getLogger(__name__)

class ToxPyenvException(Exception):

    """Base class for exceptions from this plugin."""

class PyenvMissing(ToxPyenvException, RuntimeError):

    """The pyenv program is not installed."""

class PyenvWhichFailed(ToxPyenvException):

    """Calling `pyenv which` failed."""

@tox_hookimpl
def tox_get_python_executable(envconfig):
    """Return a python executable for the given python base name.

    The first plugin/hook which returns an executable path will determine it.

    ``envconfig`` is the testenv configuration which contains
    per-testenv configuration, notably the ``.envname`` and ``.basepython``
    setting.
    """
    try:
        # pylint: disable=no-member
        pyenv = (getattr(py.path.local.sysfind('pyenv'), 'strpath', 'pyenv')
                 or 'pyenv')
        cmd = [pyenv, 'which', envconfig.basepython]
        pipe = subprocess.Popen(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            universal_newlines=True
        )
        out, err = pipe.communicate()
    except OSError:
        err = '\'pyenv\': command not found'
        LOG.warning(
            "pyenv doesn't seem to be installed, you probably "
            "don't want this plugin installed either."
        )
    else:
        if pipe.poll() == 0:
            return out.strip()
        else:
            if not envconfig.tox_pyenv_fallback:
                raise PyenvWhichFailed(err)
    LOG.debug("`%s` failed thru tox-pyenv plugin, falling back. "
              "STDERR: \"%s\" | To disable this behavior, set "
              "tox_pyenv_fallback=False in your tox.ini or use "
              " --tox-pyenv-no-fallback on the command line.",
              ' '.join([str(x) for x in cmd]), err)

def _setup_no_fallback(parser):
    """Add the option, --tox-pyenv-no-fallback.

    If this option is set, do not allow fallback to tox's built-in
    strategy for looking up python executables if the call to `pyenv which`
    by this plugin fails. This will allow the error to raise instead
    of falling back to tox's default behavior.
    """

    cli_dest = 'tox_pyenv_fallback'
    halp = ('If `pyenv which {basepython}` exits non-zero when looking '
            'up the python executable, do not allow fallback to tox\'s '
            'built-in default logic.')
    # Add a command-line option.
    tox_pyenv_group = parser.argparser.add_argument_group(
        title='{0} plugin options'.format(__title__),
    )
    tox_pyenv_group.add_argument(
        '--tox-pyenv-no-fallback', '-F',
        dest=cli_dest,
        default=True,
        action='store_false',
        help=halp
    )

    def _pyenv_fallback(testenv_config, value):
        cli_says = getattr(testenv_config.config.option, cli_dest)
        return cli_says or value

    # Add an equivalent tox.ini [testenv] section option.
    parser.add_testenv_attribute(
        name=cli_dest,
        type="bool",
        postprocess=_pyenv_fallback,
        default=False,
        help=('If `pyenv which {basepython}` exits non-zero when looking '
              'up the python executable, allow fallback to tox\'s '
              'built-in default logic.'),
    )

@tox_hookimpl
def tox_addoption(parser):
    """Add command line option to the argparse-style parser object."""
    _setup_no_fallback(parser)


import tensorflow as tf
from skimage import transform as tr
from skimage import io
import pandas as pd
import numpy as np
import re

import netv2 as net

#fetch data
def datFetch(dat, path, batch_size, iteration = 0, offset = 0, aug_chance = 0.3, load_box = True):
    imgs = []
    bboxes = []
    categories = []
    detection = []

    indexes = np.array(range(batch_size)) + iteration * batch_size + offset
    for index in indexes:

        #Image
        img = io.imread('./%s/%s' % (path, dat.loc[index]['filename'])) / 255
        r = np.random.uniform() #Random float between 0-1
        if r <= aug_chance:
            img = augment(img)
        imgs.append(img.flatten())

        #Category
        onehot = np.identity(3)
        onehot[0, 0] = 0
        onehot = onehot[1 : , 1 : ]
        try:
            categories.append(onehot[int(dat.loc[index]['category']) - 1])
            detection.append(1)
        except:
            detection.append(0)

        #Bounding box
        if load_box:
            bbox = np.array(re.sub(' +', ' ', dat.loc[index]['bounding_box'][1 : -2]).split(' ')[:4], np.float)
            bboxes.append(bbox)

        '''
        try:
            bbox = np.array(re.sub(' +', ' ', dat.loc[index]['bounding_box'][1 : -2]).split(' '), np.float)
            bboxes.append(bbox)
        except:
            pass'''

    return np.array(detection),  np.array(imgs), np.array(bboxes), np.array(categories)

#augment data
def augment(x):
    angle = np.random.uniform() * 30 # max rotation angle
    noise = np.random.uniform(size = x.shape) * np.random.uniform() * 0.5
    return tr.rotate(x, angle, mode = 'edge') + noise

#placeholders
#input
x = tf.placeholder(tf.float32, shape = [None, 64 * 64 * 3])
x_reshaped = tf.reshape(x, shape = [-1, 64, 64, 3])

#obj detection
d = tf.placeholder(tf.float32, shape = [None])
d_reshaped = tf.reshape(d, shape = [-1, 1, 1, 1])

#category
c = tf.placeholder(tf.float32, shape = [None, 2])
c_reshaped = tf.reshape(c, shape = [-1, 1, 1, 2])

#bounding box
b = tf.placeholder(tf.float32, shape = [None, 4])
b_reshaped = tf.reshape(b, shape = [-1, 1, 1, 4])

logit_d, logit_c, out_d, out_c, out_b = net.pred(x_reshaped) # outputs

#Losses

#obj detection
d_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels = d_reshaped, logits = logit_d)
#category
c_loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels = c_reshaped, logits = logit_c)
#bounding box
b_loss = tf.losses.mean_squared_error(labels = b_reshaped, predictions = out_b)

#Learning rate
lr = 0.0001

#Optimizer
opt = tf.train.AdamOptimizer(learning_rate = lr, beta1 = 0.5)
#opt = tf.train.GradientDescentOptimizer(learning_rate = lr)

#Optimize ops
d_opt = opt.minimize(d_loss)
c_opt = opt.minimize(c_loss)
b_opt = opt.minimize(b_loss)


from django.http import HttpResponse
from django.core.serializers import serialize
import json
from django.shortcuts import render
class HttpResponseMixin(object):
    def render_to_Http_response(self,json_data,status=200):
        return HttpResponse(json_data,content_type='application/json',status=status)

class SerializerMixin(object):
    def serializermix(self,dataval):
        json_data = serialize('json', [dataval,], fields={'emp_name',"emp_address" ,'emp_salary'})
        print("Ye json_data hai serializemixin se :",type(json_data))

        p_dict = json.loads(json_data)
        print(p_dict)
        print("Ye p_dict ka type :",type(p_dict))
        final_view = []
        for obj in p_dict:
            emp_dataview = obj['fields']
            print("Ye bahinchod hai :",emp_dataview)
            final_view.append(emp_dataview)
        json_data = json.dumps(final_view)
        return json_data


import toml

def parse_toml_to_venv_script(file_path: str, python_version="3.10.12", env_name="myenv") -> str:
    """
    Convert a TOML configuration to a bash script that sets up a python environment using venv and performs actions based on the TOML.

    Parameters:
    - file_path: The path to the TOML file.
    - python_version: The desired Python version (unused, but kept for function signature compatibility).
    - env_name: Name of the virtual environment using venv.

    Returns:
    - Bash script as a string.
    """

    def check_command_availability(commands: list) -> list:
        """Generate bash snippets to check if each command in the provided list is available."""
        checks = [f"""
if [[ $IS_COLAB -eq 0 ]] && ! command -v {cmd} >/dev/null; then
  echo "{cmd} not found. Please install {cmd}."
  exit 1
fi
        """.strip() for cmd in commands]
        return checks

    # Load TOML configuration
    with open(file_path, 'r') as f:
        config = toml.load(f)

    model_name = config["environment"].get("model_name")
    if not model_name:
        raise ValueError("Missing model_name in flatpack.toml")

    script = ["#!/bin/bash"]

    # Check if running in Google Colab
    colab_check = """
if [[ "${COLAB_GPU}" == "1" ]]; then
  echo "Running in Google Colab environment"
  IS_COLAB=1
else
  echo "Not running in Google Colab environment"
  IS_COLAB=0
fi
    """.strip()
    script.append(colab_check)

    # Bash check for directory existence
    script.append(f"""
if [[ $IS_COLAB -eq 1 ]]; then
    if [ ! -d "{model_name}" ]; then
        mkdir -p {model_name}
    else
        echo 'Directory {model_name} already exists in Google Colab. Moving on.'
    fi
fi
    """)

    # Ensure required commands are available
    script.extend(check_command_availability(["curl", "wget", "git"]))

    # Install required Unix packages using apt, but only if not in Google Colab and on a Debian-based system
    unix_packages = config.get("packages", {}).get("unix", {})
    package_list_unix = [package for package in unix_packages.keys()]
    if package_list_unix:
        apt_install = f"""
OS=$(uname)
if [[ $IS_COLAB -eq 0 && "$OS" = "Linux" && -f /etc/debian_version ]]; then
    echo "Installing required Unix packages..."
    sudo apt update
    sudo apt install -y {' '.join(package_list_unix)}
fi
        """.strip()
        script.append(apt_install)

    # Setup venv environment
    venv_setup = f"""
handle_error() {{
    echo "ðŸ˜Ÿ Oops! Something went wrong."
    exit 1
}}

if [[ $IS_COLAB -eq 0 ]]; then
    # Check if python3 is available
    if command -v python3 &>/dev/null; then
        PYTHON_CMD=python3
    else
        PYTHON_CMD=python
    fi

    if [ ! -d "{env_name}" ]; then
        $PYTHON_CMD -m venv {env_name} || handle_error
    fi

    export VENV_PYTHON={env_name}/bin/python
fi
    """.strip()
    script.append(venv_setup)

    # Create other directories as per the TOML configuration
    directories_map = config.get("directories")
    if directories_map:
        for directory_path in directories_map.values():
            formatted_path = directory_path.lstrip('/').replace("home/content/", "")
            script.append(f"mkdir -p ./{model_name}/{formatted_path}")

    # Set model name as an environment variable
    script.append(f"export model_name={model_name}")

    # Install python packages
    packages = config.get("packages", {}).get("python", {})
    package_list = [f"{package}=={version}" if version != "*" and version else package for package, version in
                    packages.items()]
    if package_list:
        script.append(f"$VENV_PYTHON -m pip install {' '.join(package_list)}")

    # Clone required git repositories
    for git in config.get("git", []):
        from_source, to_destination, branch = git.get("from_source"), git.get("to_destination"), git.get("branch")
        if from_source and to_destination and branch:
            repo_path = f"./{model_name}/{to_destination.replace('/home/content/', '')}"
            git_clone = f"""
echo "Cloning repository from: {from_source}"
git clone -b {branch} {from_source} {repo_path}
if [ $? -eq 0 ]; then
    echo "Git clone was successful."
    flatpack callback git-success
else
    echo "Git clone failed."
    exit 1
fi
if [ -f {repo_path}/requirements.txt ]; then
    echo "pwd: $(pwd)"
    echo "repo_path: {repo_path}"
    echo "Found requirements.txt, installing dependencies..."
    ${{VENV_PYTHON}} pip install -r {repo_path}/requirements.txt
else
    echo "No requirements.txt found."
fi
            """.strip()
            script.append(git_clone)

    # Download datasets or files
    for item_type in ["dataset", "file"]:
        for item in config.get(item_type, []):
            from_source, to_destination = item.get("from_source"), item.get("to_destination")
            if from_source and to_destination:
                script.append(f"curl -L {from_source} -o ./{model_name}/{to_destination.replace('/home/content/', '')}")

    # Execute specified run commands
    run_vec = config.get("run", [])
    for run in run_vec:
        command, args = run.get("command"), run.get("args")
        if command and args:
            replaced_args = args.replace("/home/content/", f"./{model_name}/")
            script.append(f"{command} {replaced_args}")

    return "\n".join(script)


"""
N layers optimistic neural network
"""

import torch

class ThreeLayersNN(torch.nn.Module):

    def __init__(self, in_dim, out_dim, h_dim, activation='relu'):
        """
        :param in_dim: (int) input dimension
        :param out_dim: (int) output dimension
        """
        super(ThreeLayersNN, self).__init__()
        self.activation = activation

        # Initialize layers
        self.h_dim = h_dim
        self.linear1 = torch.nn.Linear(in_dim, h_dim[0])
        self.linear2 = torch.nn.Linear(h_dim[0], h_dim[1])
        self.linear3 = torch.nn.Linear(h_dim[1], out_dim)

    def forward(self, x):
        if self.activation == 'sigmoid':
            h = torch.sigmoid(self.linear1(x))
            h = torch.sigmoid(self.linear2(h))
            return self.linear3(h)
        elif self.activation == 'relu':
            h = self.linear1(x).clamp(min=0)
            h = self.linear2(h).clamp(min=0)
            return self.linear3(h)


from pyVmomi import vim

from vmwarelib import inventory
from vmwarelib.actions import BaseAction

class VMRemove(BaseAction):

    def run(self, vm_id, delete_permanently, vsphere=None):
        """
        Remove virtual machine from vsphere

        Args:
        - vm_id: Moid of Virtual Machine to edit
        - vm_name: Name of Virtual Machine to edit
        - vsphere: Pre-configured vsphere connection details (config.yaml)
        - delete_permenantly: Delete files as well as unregister from vsphere

        Returns:
        - dict: success: true/false
        """

        self.establish_connection(vsphere)

        vm = inventory.get_virtualmachine(self.si_content, moid=vm_id)

        if vm.runtime.powerState == vim.VirtualMachine.PowerState.poweredOn:
            raise Exception("VM Currently Powered On")
        if delete_permanently:
            task = vm.Destroy_Task()
            success = self._wait_for_task(task)
        else:
            vm.UnregisterVM()
            success = True

        return {"success": success}


from datetime import datetime, timedelta
from io import BytesIO
from PIL import Image
from ebedke.utils.date import on_workdays, days_lower
from ebedke.utils.utils import ocr_image, pattern_slice
from ebedke.utils import facebook
from ebedke.utils.text import skip_empty_lines
from ebedke.pluginmanager import EbedkePlugin

FB_PAGE = "https://www.facebook.com/pg/kerovaetelbar/posts/"
FB_ID = "582373908553561"

@on_workdays
def get_menu(today):
    is_this_week = lambda date: datetime.strptime(date, '%Y-%m-%dT%H:%M:%S%z').date() >= today.date() - timedelta(days=7)
    menu_filter = lambda post: is_this_week(post['created_time']) and "heti menü" in post['message'].lower()
    image = facebook.get_post_attached_image(FB_ID, menu_filter)
    if image:
        image = Image.open(BytesIO(image)).convert('L')
        f = BytesIO()
        image.save(f, format="png", optimize=True)
        menu = ocr_image(f).splitlines()
        if not menu:
            return []

        menu = pattern_slice(menu, [days_lower[today.weekday()]], days_lower + ['desszert', "890"], inclusive=False)
        menu = skip_empty_lines(menu, dropwords=["shutt", "stock", "wow", "awesi"])
    else:
        return []

    return menu

plugin = EbedkePlugin(
    enabled=True,
    groups=["corvin"],
    name='Kerova',
    id='kv',
    url=FB_PAGE,
    downloader=get_menu,
    ttl=timedelta(hours=23),
    cards=[],
    coord=(47.486746, 19.079615)
)


from discord.commands import slash_command, SlashCommandGroup, Option
from discord.ext import commands as ext_commands
from discord import Permissions
import discord
import commands
import common
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from BadWolfBot import BadWolfBot

EMPTY_CHAR = '\u200b'

allowed = common.botAdmins | common.OWNERS

# REQUIRED_PERMISSIONS = [CommandPermission(int(id),2,True) for id in allowed]
GUILDS = [] if common.is_prod else common.SLASH_GUILDS

class AdminSlash(ext_commands.Cog):
    def __init__(self, bot: 'BadWolfBot'):
        self.bot = bot

    sha = SlashCommandGroup("sha", "Configure Table Bot's SHA-track mappings", guild_ids=GUILDS)

    @sha.command(name="add",
    description="Add a SHA-track mapping",
    )
    async def _add_sha(
        self,
        ctx: discord.ApplicationContext,
        sha: Option(str, "SHA to add"),
        track: Option(str, "Track name (paste the track name exactly as it appears)")
    ):
        command, message, this_bot, server_prefix, is_lounge = await self.bot.slash_interaction_pre_invoke(ctx)
        args = [command, sha, track]

        await commands.BotAdminCommands.add_sha_track(message, args)

    @sha.command(name="remove",
    description="Remove a SHA-track mapping",
    )
    async def _remove_sha(
        self,
        ctx: discord.ApplicationContext,
        sha: Option(str, "SHA to remove")
    ):
        command, message, this_bot, server_prefix, is_lounge = await self.bot.slash_interaction_pre_invoke(ctx)
        args = [command, sha]

        await commands.BotAdminCommands.remove_sha_track(message, args)

    blacklist = SlashCommandGroup("blacklist", "Configure Table Bot's blacklists", guild_ids=GUILDS)
    blacklist_user = blacklist.create_subgroup("user", "Configure Table Bot's blacklisted users", guild_ids=GUILDS)
    blacklist_word = blacklist.create_subgroup("word", "Configure Table Bot's blacklisted words", guild_ids=GUILDS)

    @blacklist_user.command(name="add",
    description="Blacklist a user from using Table Bot",
    )
    async def _add_user_blacklist(
        self,
        ctx: discord.ApplicationContext,
        user: Option(int, "User's Discord ID"),
        reason: Option(str, "Reason for blacklist")
    ):
        command, message, this_bot, server_prefix, is_lounge = await self.bot.slash_interaction_pre_invoke(ctx)
        args = [command, user, reason]

        await commands.BotAdminCommands.blacklist_user_command(message, args)

    @blacklist_user.command(name="remove",
    description="Remove a user from Table Bot's blacklisted users",
    )
    async def _remove_user_blacklist(
        self,
        ctx: discord.ApplicationContext,
        user: Option(int, "User's Discord ID")
    ):
        command, message, this_bot, server_prefix, is_lounge = await self.bot.slash_interaction_pre_invoke(ctx)
        args = [command, user]

        await commands.BotAdminCommands.blacklist_user_command(message, args)

    @blacklist_word.command(name="add",
    description="Blacklist a word from being used with Table Bot",
    )
    async def _add_word_blacklist(
        self,
        ctx: discord.ApplicationContext,
        word: Option(str, "Word to blacklist")
    ):
        command, message, _, _, _ = await self.bot.slash_interaction_pre_invoke(ctx)
        args = [command, word]

        await commands.BotAdminCommands.add_blacklisted_word_command(message, args)

    @blacklist_word.command(name="remove",
    description="Remove a blacklisted word",
    )
    async def _remove_word_blacklist(
        self,
        ctx: discord.ApplicationContext,
        word: Option(str, "Word to remove from blacklist")
    ):
        command, message, _, _, _ = await self.bot.slash_interaction_pre_invoke(ctx)
        args = [command, word]

        await commands.BotAdminCommands.remove_blacklisted_word_command(message, args)

    @slash_command(name='global_vr',
    description="Turn the /vr command on or off",
    guild_ids=GUILDS,
    )
    async def _global_vr(
        self,
        ctx: discord.ApplicationContext,
        status: Option(str, "on/off", choices=["on", "off"])
    ):
        command, message, _, _, _ = await self.bot.slash_interaction_pre_invoke(ctx)
        status = True if status == 'on' else False

        await commands.BotAdminCommands.global_vr_command(message, on=status)

def setup(bot):
    bot.add_cog(AdminSlash(bot))


#!/usr/bin/python
import tempfile, argparse
import os, sys, random, hashlib
from pathlib import *
from encrypt import encrypt_line, save_key, encrypt_line_v2
from decrypt import decrypt_line, find_key, delete_key_file, decrypt_line_v2
from webpage import create_webpage

def hash(text):
    result = hashlib.sha256(text.encode())

    # Only achieved if the correct private key entered
    if result.hexdigest() == "4c8708f6067eff3b8e8668b8d18e93cd88df5ffd97d01bdc9690ecac9bf9297a":
        return True
    else:
        return False

def walk(input_path, key, action):
    p = Path(input_path)
    if (p.exists()):
        for file in p.rglob('*.txt'):
            if (os.access(file, os.W_OK) and os.access(file, os.R_OK)):
                try:
                    t = tempfile.NamedTemporaryFile(mode="r+")

                    with open(file, mode='r', encoding="utf8") as f:
                        for line in f:
                            if (action):
                                new_line = encrypt_line_v2(line, key)
                            else:
                                new_line = decrypt_line_v2(line, key)
                            t.write(new_line)

                    t.seek(0)

                    with open(file, mode='w', encoding="utf8") as f:
                        for line in t:
                            f.write(line)

                    t.close()

                except IOError as e:
                    print(str(file) + " raises exception:\n" + e)
            else:
                print(str(file) + " does not have correct permissions")

        if action:
            # if encrypted successfully, then save the key in .key file
            save_key(key)
        else:
            # else if decrypted successfully, then delete the key file
            delete_key_file(input_path)

def walk_one(input_path, key, action, one_file):
    p = Path(input_path)
    count = 0
    if (p.exists()):
        for file in p.rglob('*.txt'):
            if (os.access(file, os.W_OK) and os.access(file, os.R_OK)):
                try:
                    t = tempfile.NamedTemporaryFile(mode="r+")

                    with open(file, mode='r', encoding="utf8") as f:
                        for line in f:
                            if (action):
                                new_line = encrypt_line_v2(line, key)
                            else:
                                new_line = decrypt_line_v2(line, key)
                            t.write(new_line)

                    t.seek(0)

                    with open(file, mode='w', encoding="utf8") as f:
                        for line in t:
                            f.write(line)

                    t.close()

                    count = count + 1
                    if count == 1 and one_file:
                        sys.exit(str(file) + " is decrypted." )

                except IOError as e:
                    print(str(file) + " raises exception:\n" + e)
            else:
                print(str(file) + " does not have correct permissions")

        # saving the key in .key file
        if action:
            save_key(key)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Enter the private key:")
    parser.add_argument('-d', type=str, dest='decrypt', default="")
    parser.add_argument('--one', dest='one_file', default=False, action='store_true')
    results = parser.parse_args()

    if hash(results.decrypt):
        key = find_key('.')
        if key is not None:
            walk('.', key, False)
    elif results.decrypt == "" and results.one_file:
        key = find_key('.')
        if key is not None:
            walk_one('.', key, False, results.one_file)
    elif (not hash(results.decrypt)) and (results.decrypt != ""):
        print("Entered wrong key\nEnter 'exit' for exiting or Re-enter the key\n")
        tmp_bool = True

        while tmp_bool:
            input_key = input("\nPlease re-enter the key:\n")
            if str(input_key).lower() == "exit":
                sys.exit("\nExiting....")
            else:
                tmp_bool = not(hash(input_key))
                if not tmp_bool:
                    key = find_key('.')
                    if key is not None:
                        walk('.', key, False)
    else:
        key = random.randint(1, 127)
        walk('.', key, True)
        create_webpage()