# pylint: disable=R0904,C0103
""":class:`geoutils.GdeltConfig` tests.

"""
import unittest2
import tempfile
import os

import geoutils
from geosutils.files import (copy_file,
                             remove_files)
import geolib_mock

class TestGdeltDaemon(unittest2.TestCase):
    """:class:`geoutils.GdeltDaemon` test cases.
    """
    @classmethod
    def setUpClass(cls):
        conf = os.path.join('geoutils',
                            'tests',
                            'files',
                            'proxy.properties')
        cls._mock = geolib_mock.MockServer(conf)
        cls._mock.start()

        cls._conf = geoutils.GdeltConfig()
        cls._conf.threads = 1
        cls._conf.inbound_dir = tempfile.mkdtemp()

        cls._gdelt_spatial_index_table_name = 'gdelt_spatial_index'
        cls._audit_table_name = 'audit'

    def setUp(self):
        self._gdeltd = geoutils.GdeltDaemon(pidfile=None, conf=self._conf)

        self._ds = geoutils.Datastore()
        self._ds.host = 'localhost'
        self._ds.port = 42425
        self._ds.user = 'root'

    def test_init(self):
        """Initialise a GdeltDaemon object.
        """
        msg = 'Object is not a geoutils.GdeltDaemon'
        self.assertIsInstance(self._gdeltd, geoutils.GdeltDaemon, msg)

    def test_start_dry_run(self):
        """GdeltDaemon dry run.
        """
        self._ds.connect()
        self._ds.init_table(self._gdelt_spatial_index_table_name)
        self._ds.init_table(self._audit_table_name)

        gdelt_dir = os.path.join('geoutils', 'daemon', 'tests', 'files')
        gdelt_file = '20141124.export.CSV.zip'
        target_file = os.path.join(self._conf.inbound_dir, gdelt_file)
        copy_file(os.path.join(gdelt_dir, gdelt_file), target_file)

        old_dry = self._gdeltd.dry

        self._gdeltd.dry = True
        self._gdeltd._start(self._gdeltd.exit_event)

        # Clean up.
        self._gdeltd.dry = old_dry
        self._gdeltd.exit_event.clear()
        remove_files(target_file)
        self._ds.delete_table(self._gdelt_spatial_index_table_name)
        self._ds.delete_table(self._audit_table_name)

    def tearDown(self):
        del self._gdeltd
        del self._ds

    @classmethod
    def tearDownClass(cls):
        cls._mock.stop()

        del cls._gdelt_spatial_index_table_name
        del cls._audit_table_name

        os.removedirs(cls._conf.inbound_dir)
        del cls._conf


#!/usr/bin/env python3

"""
    Server side of Assignment 1 for ECE 4564.
    Handles:
    1. Receiving question from client.
    2. Speaking question.
    3. Sending a receiving from WolframAlpha API.
    4. Sending answer to client.
"""

import os
import sys
import argparse
import socket

from cryptography.fernet import Fernet
import wolframalpha
import hashlib
import pickle
from gtts import gTTS

from serverKeys import wolfram_alpha_appid

def checkpoint(message):
    """Prints [Checkpoint] <message>"""
    print("[Checkpoint] {}".format(message))

def encrypt_data(data, key):
    """Encrypt given data and returns key, ciphertest, and md5"""
    # Has to be the same key because the client only has the key it
    #     sent the data with
    cipher_suite = Fernet(key)
    # Defaults to utf-8
    bytedata = data.encode()
    # Encrypt WA Answer
    ciphertext = cipher_suite.encrypt(bytedata)

    # Determine the md5 of ciphertext
    temp = hashlib.md5()
    temp.update(ciphertext)
    md5 = temp.digest()

    return ciphertext, md5

def ask_wa(message):
    """Asks Wolfram Alpha messasge"""
    checkpoint("Sending question to Wolframalpha: {}".format(message))

    # Get answer from WA
    client = wolframalpha.Client(wolfram_alpha_appid)
    res = client.query(message)

    # Handle bad queries to WA
    try:
        answer = next(res.results).text
    except Exception as ex:
        answer = "Wolfram Alpha was unable to find an answer for {}. \
            Please try something else.".format(message)

    checkpoint("Received answer to Wolframalpha: {}".format(answer))

    return answer

def speak(message):
    """Speaks given message"""
    checkpoint("Speaking: {}".format(message))

    tts = gTTS(text=message, lang='en')
    tts.save("./saythis.mp3")
    os.system("mplayer ./saythis.mp3 > /dev/null 2>&1")

def decrypt_data(data):
    """Verify md5 and decrypt data"""
    # Data should be formatted as the tuple: (key, ciphertext, md5)

    # Extract key & decrypt data
    key = data[0]
    cipher_suite = Fernet(key)
    plaintext = cipher_suite.decrypt(data[1])

    # Get md5 of data
    temp = hashlib.md5()
    temp.update(data[1])
    md5 = temp.digest()

    return key, plaintext, md5

def accept_connections(socket, size):
    """Repeatedly accepts and handles new client connections"""
    while 1:
        client, address = socket.accept()
        client_addr = client.getpeername()[0]
        client_port = client.getpeername()[1]
        checkpoint("Accepted client connection from {} on port {}".format(
            client_addr, client_port))

        data = client.recv(size)
        if data:
            checkpoint("Received data: {}".format(data))

            # Decrypt data
            decryptedData = pickle.loads(data)
            recv_key, recv_plaintext, recv_md5 = decrypt_data(decryptedData)

            # Check if md5 is valid
            if decryptedData[2] == recv_md5:
                checkpoint("Checksum is VALID")
            else:
                checkpoint("Checksum is INVALID")

            checkpoint("Decrypt: Using Key: {} | Plaintext: {}"
                .format(recv_key, recv_plaintext, recv_md5))

            # Speak data
            speak(recv_plaintext.decode())

            # Ask WA
            answer = ask_wa(recv_plaintext.decode())

            # Encrypt response
            send_ciphertext, send_md5 = encrypt_data(answer, recv_key)
            checkpoint("Encrypt: Generated Key: {} | Ciphertext: {}"
                .format(recv_key, send_ciphertext))
            checkpoint("Generated MD5 Checksum: {}".format(send_md5))

            # Send response back to client
            dataTup = (recv_key, send_ciphertext, send_md5)
            send_data = pickle.dumps(dataTup)
            client.send(send_data)
            checkpoint("Sending data: {}".format(send_data))

            client.close()

def main():
    """Main function to manager the server"""
    # Parse command line args
    parser = argparse.ArgumentParser(description='Prossesses arguments')
    parser.add_argument('-p', help='Set the server port.')
    parser.add_argument('-b', help='Set the backlog size.')
    parser.add_argument('-z', help='Set the socket size')

    args = parser.parse_args()
    if args.p == None:
        print('Please set server port with the -p flag.')
        sys.exit(1)
    if args.b == None:
        print('Please set backlog size with the -b flag.')
        sys.exit(1)
    if args.z == None:
        print('Please set socket size with the -z flag.')
        sys.exit(1)

    host = '0.0.0.0'

    # Copy args into vars
    try:
        port =      int(args.p)
        backlog =   int(args.b)
        size =      int(args.z)
    except Exception as ex:
        print(ex)
        sys.exit(1)

    # Setup socket
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind((host,port))
    checkpoint("Created socket at {} on port {}".format(host, port))

    # Start listening
    checkpoint("Listening for client connections")
    s.listen(backlog)

    # Start accepting connections
    accept_connections(s, size)

main()


import xarray as xr
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import ticker
import datetime as datetime
from common import time_mean_to_orca

def add_slice(ax, ds, time, depth):

    ds = ds.isel(time_counter=time, deptht=depth)

    lev = np.linspace(-2,2,11)
    p = ax.pcolor(ds.nav_lon, ds.nav_lat, ds.vozocrtx, vmin=-0.4, vmax=0.4,
                  cmap=plt.cm.inferno)

    return p

def plot(orca_path, sochic_path, model, depth0, depth1):
    '''
    main plotting routine adding six time slices over two depths
    '''

    # intialise plots
    fig, axs = plt.subplots(2,6, figsize=(6.5, 2), dpi=300)
    plt.subplots_adjust(bottom=0.15, top=0.98, right=0.90, left=0.10,
                        wspace=0.1, hspace=0.1)

    # load data
    orca   = xr.open_dataset(orca_path, decode_cf=False)
    sochic = xr.open_dataset(sochic_path, decode_cf=True)

    # translate orca dates to cf
    orca.time_counter.attrs['units'] = 'seconds since 1900-01-01'
    orca.time_centered.attrs['units'] = 'seconds since 1900-01-01'
    orca = xr.decode_cf(orca)

    #sochic['time_counter'] = sochic.indexes['time_counter'].to_datetimeindex()

    #if model == 'orca':
    m = orca
    #if model == 'sochic':
    #    # align time steps
    #m = time_mean_to_orca(orca, sochic)

    # plot six time steps at depth0
    for i, ax in enumerate(axs[0]):
        print ('a', i)
        add_slice(ax, m, i, depth0)

    # plot six time steps at depth1
    for i, ax in enumerate(axs[1]):
        print ('b', i)
        p = add_slice(ax, m, i, depth1)

    pos = axs[0,1].get_position()
    cbar_ax = fig.add_axes([0.91, pos.y0, 0.02, pos.y1 - pos.y0])
    cbar = fig.colorbar(p, cax=cbar_ax)
    #cbar.locator = ticker.MaxNLocator(nbins=3)
    #cbar.update_ticks()
    #cbar.ax.text(4.5, 0.5, labels[i], fontsize=8, rotation=90,
    cbar.ax.text(4.5, 0.5, 'temperature', fontsize=8, rotation=90,
                 transform=cbar.ax.transAxes, va='center', ha='right')
    for ax in axs[0,:]:
        ax.set_xticks([])
    for ax in axs[0,1:]:
        ax.set_yticks([])
    for ax in axs[1,1:]:
        ax.set_yticks([])
    for ax in axs.flatten():
        ax.set_aspect('equal')

    plt.savefig(model + 'orca_u_orca_mean.png')

if __name__ == '__main__':
    model = 'EXP106'
    orca_path = '../processORCA12/DataOut/ORCA_PATCH_U.nc'
    #orca_path = '../processORCA12/DataOut/ORCA0083-N06_T_conform.nc'
    sochic_path = '../Output/EXP106/SOCHIC_PATCH_1h_20150106_20150118_grid_U.nc'
   #sochic_path = '../Output/EXP74/SOCHIC_PATCH_1h_20150106_20150130_grid_T.nc'
    #sochic_path = '../Output/EXP60/SOCHIC_PATCH_1h_20150101_20150121_grid_T.nc'

    #SOCHIC_PATCH_1h_20150101_20150130_grid_T.nc'
    #plot(orca_path, sochic_path, 'orca', depth0=0, depth1=20)
    plot(orca_path, sochic_path, model, depth0=0, depth1=20)


import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from tensorflow.python.ops import rnn, rnn_cell
from data_pre import *
from lstm import *
from nltk import sent_tokenize, word_tokenize
from nltk.stem.snowball import SnowballStemmer as stemmer
import sys

def test_lstm(sent ,prediction):
	sent = np.array(sent_to_vec(sent[:-1], lexicon)).reshape((-1, n_chunks, chunk_size))
	with tf.Session() as sess:
		saver.restore(sess, 'model.ckpt')
		p = tf.argmax(prediction,1)
		pre = p.eval(feed_dict={x:sent}, session=sess)
	return pre

# sent = ['to', 'contribute', 'any', 'really', 'good', 'enigmas']
# sent = ['.--', 'Fancying', 'you', 'to', 'have', 'fathomed']
# print(gre[test_lstm(sent, predict)[0]])

input = "All children, except one, grow up. They soon know that they will grow\
up, and the way Wendy knew was this. One day when she was two years old\
she was playing in a garden, and she plucked another flower and ran with\
it to her mother. I suppose she must have looked rather delightful, for\
Mrs. Darling put her hand to her heart and cried, “Oh, why can’t you\
remain like this for ever!” This was all that passed between them on\
the subject, but henceforth Wendy knew that she must grow up. You always\
know after you are two. Two is the beginning of the end."

stem = stemmer("english")
def changeable():
	# read in the changeable word vocabulary
	return set(line.strip('\n') for line in open('synonym.txt'))

change = changeable()
def replacable(word):
	word = stem.stem(word)
	if word in change:
		return True
	else: return False

def complifier2():
	input = "Of course they lived at 14 [their house number on their street], and\
until Wendy came her mother was the chief one. She was a lovely lady,\
with a romantic mind and such a sweet mocking mouth. Her romantic\
mind was like the tiny boxes, one within the other, that come from the\
puzzling East, however many you discover there is always one more; and\
her sweet mocking mouth had one kiss on it that Wendy could never get,\
though there it was, perfectly conspicuous in the right-hand corner."
	sents = ""
	output = ""
	for line in input:
		sents += line.replace('\n',' ')
	sents = sent_tokenize(sents)
	for sent in sents:
		sent = word_tokenize(sent)
		for word in sent:
			if replacable(word):
				i = sent.index(word)
				sixgram = []
				count = 5
				while count > 0:
					if sent[i-count]:
						sixgram.append(sent[i-count])
					else: sixgram.append('un_k')
					count = count - 1
				sixgram.append(word)
				replaceword = gre[test_lstm(sixgram, predict)[0]]
				output += '
'+word+''+replaceword+'
'
					# add something to implement the tooltip
			else:
				# print(word)
				output = output + word + " "
				# add something to implement to tooltip
	return output

# def test_argu(sys.argv[1]):
# 	print(sys.argv[1])
# print(complifier2(sys.argv[1]))
output2 = complifier2()
# if __name__=='__main__':
# 	sys.exit(main(sys.argv[1]))


from moviepy.editor import VideoFileClip, concatenate_videoclips
from moviepy.video.fx.resize import resize
import os
from os.path import isfile, join
import random
import shutil
from collections import defaultdict

VideoFileClip.resize = resize

def extractAcc(filepath):
    try:
        s = filepath.split("/")[-1].split("-")
        acc = "-".join(s[1:(2+(len(s) - 4))])
        return acc
    except:
        return ""

# generateTimeRange converts float seconds to a range of form @MM:SS
def generateTimeRange(duration, clipDuration):
    preHour = int(duration / 60)
    preMin = int(duration % 60)
    preTime = str(preHour // 10) + str(preHour % 10) + ":" + str(preMin // 10) + str(preMin % 10)

    duration += clipDuration
    postHour = int(duration / 60)
    postMin = int(duration % 60)
    postTime = str(postHour // 10) + str(postHour % 10) + ":" + str(postMin // 10) + str(postMin % 10)

    #return "@" + preTime + " - " + "@" + postTime
    return "@" + preTime

# makeCompilation takes videos in a folder and creates a compilation with max length totalVidLength
def makeCompilation(path = "./",
                    introName = '',
                    outroName = '',
                    totalVidLength = 10*60,
                    maxClipLength = 20,
                    minClipLength = 5,
                    outputFile = "output.mp4"):

    allVideos = []
    seenLengths = defaultdict(list)
    totalLength = 0
    for fileName in os.listdir(path):

        filePath = join(path, fileName);
        if isfile(filePath) and fileName.endswith(".mp4"):
            print(fileName)
            if os.stat(filePath).st_size < 5000:
                continue

            # Destination path
            clip = VideoFileClip(filePath)
            clip = clip.resize(width=1920)
            clip = clip.resize(height=1080)
            duration = clip.duration
            print(duration)
            if duration <= maxClipLength and duration >= minClipLength:
                allVideos.append(clip)
                seenLengths[duration].append(fileName)
                totalLength += duration

    print("Total Length: " + str(totalLength))

    random.shuffle(allVideos)

    duration = 0
    # Add intro vid
    videos = []
    if introName != '':
        introVid = VideoFileClip("./" + introName)
        videos.append(introVid)
        duration += introVid.duration

    description = ""
    # Create videos
    for clip in allVideos:
        timeRange = generateTimeRange(duration, clip.duration)
        acc = extractAcc(clip.filename)
        description += timeRange + " : @" + acc + "\n"
        duration += clip.duration
        videos.append(clip)
        print(duration)
        if duration >= totalVidLength:
            # Just make one video
            break

    # Add outro vid
    if outroName != '':
        outroVid = VideoFileClip("./" + outroName)
        videos.append(outroVid)

    finalClip = concatenate_videoclips(videos, method="compose")

    audio_path = "/tmp/temoaudiofile.m4a"

    #print(description)
    # Create compilation
    finalClip.write_videofile(outputFile, threads=8, temp_audiofile=audio_path, remove_temp=True, codec="libx264", audio_codec="aac")

    return description

if __name__ == "__main__":
    makeCompilation(path = "/Users/nathanan/Documents/YOUTUBE/AutomatedChannel/Videos/Memes/",
                    introName = "intro_vid.mp4",
                    outroName = '',
                    totalVidLength = 10*60,
                    maxClipLength = 20,
                    outputFile = "outputseq.mp4")


#!/usr/bin/env python

import json
import os
from argparse import ArgumentParser, RawTextHelpFormatter
from pathlib import Path
from textwrap import dedent

from Hive.utils.log_utils import (
    get_logger,
    add_verbosity_options_to_argparser,
    log_lvl_from_verbosity_args,
)

DESC = dedent(
    """
    Run nnDetection pre-processing, data analysis, and image data unpacking.
    The CL scripts called are  ``nndet_prep`` and ``nndet_unpack``, with the arguments extracted from the given configuration file.
    """  # noqa: E501
)
EPILOG = dedent(
    """
    Example call:
    ::
        {filename} --config-file /PATH/TO/CONFIG_FILE.json
    """.format(  # noqa: E501
        filename=Path(__file__).stem
    )
)

def get_arg_parser():
    pars = ArgumentParser(description=DESC, epilog=EPILOG, formatter_class=RawTextHelpFormatter)

    pars.add_argument(
        "--config-file",
        type=str,
        required=True,
        help="File path for the configuration dictionary, used to retrieve experiment variables (**Task_ID**, **results_folder**, ...) ",
        # noqa: E501
    )

    pars.add_argument(
        "--n-workers",
        type=str,
        default=None,
        help="Number of parallel processes used when pre-processing and unpacking the image data (Default: ``N_THREADS``)",
    )

    add_verbosity_options_to_argparser(pars)

    return pars

def main():
    parser = get_arg_parser()
    arguments, unknown_arguments = parser.parse_known_args()
    args = vars(arguments)

    logger = get_logger(  # NOQA: F841
        name=Path(__file__).name,
        level=log_lvl_from_verbosity_args(args),
    )
    config_file = args["config_file"]

    n_workers = "1"
    if args["n_workers"] is None:
        if "N_THREADS" in os.environ is not None:
            n_workers = str(os.environ["N_THREADS"])
    else:
        n_workers = str(args["n_workers"])

    with open(config_file) as json_file:
        data = json.load(json_file)

        arguments = [data["Task_ID"], "-np", n_workers, "-npp", n_workers]

        os.environ["det_data"] = data["base_folder"]
        os.environ["OMP_NUM_THREADS"] = "1"

        os.environ["nnUNet_def_n_proc"] = n_workers
        os.environ["det_models"] = data["results_folder"]
        arguments.extend(unknown_arguments)
        os.system("nndet_prep " + " ".join(arguments))
        os.system(
            "nndet_unpack {} {}".format(
                str(
                    Path(os.environ["det_data"]).joinpath(
                        "Task{}_{}".format(data["Task_ID"], data["Task_Name"]), "preprocessed", "D3V001_3d", "imagesTr"
                    )
                ),
                n_workers,
            )
        )

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
"""
Import terrain height data from OS OpenData files

Data requested from OS OpenData:
OS Terrain 50 - GB | Data type: Grid | Supply format: ASCII GRID AND GML (GRID)
From: https://www.ordnancesurvey.co.uk/opendatadownload/products.html

This provides a .zip folder, which this project expects to be unzipped into:
    "./OS - terr50_gagg_gb"

Inside that is data/ and a license
Inside data/ are a lot of folders for the Ordnance Survey national grid

In each of those folders are several zip files, which contain the data for
each of the grid squares. The important one of these is the .asc file
"""

from PIL import Image
import zipfile
import os
from shutil import rmtree
from math import floor
import numpy as np

class HeightCell:
    """
    A single cell of the height map from Ordnance Survey's site
    """
    def __init__(self, dimensions, heights, exclude=[]):
        """
        Initialise this HeightCell with the properties passed to it
        dimensions is (cols, rows, xcorner, ycorner, cellsize)
        """
        cols, rows, xcorner, ycorner, cellsize = dimensions

        self.xsize = cols
        self.ysize = rows
        self.xcorner = xcorner  # Lower left corner
        self.ycorner = ycorner  # Lower left corner
        self.size = cellsize
        self.heights = heights
        self.exclude = exclude

        valid_values = list(filter(lambda x: x is not None, self.flattened))
        self.max = max(valid_values)
        self.min = min(valid_values)

    @property
    def flattened(self):
        """
        Return a flattened list of the heights in the cell
        Yes, I'm aware of the irony in the name
        """
        flat = [item for sublist in self.heights for item in sublist]
        return [h if h not in self.exclude else None for h in flat]

def importAsc(asc_file_name):
    """
    Import a .asc file from Ordnance Survey as a HeightCell
    """

    with open(asc_file_name, "r") as asc_file:
        lines = asc_file.readlines()

    # First 5 lines are dimensions
    dimensions = (int(l.split(" ")[-1]) for l in lines[:5])

    # Lines after this are the height data
    heights = [list(map(float, l.rstrip().split(" "))) for l in lines[5:]]

    # Some values may need to be excluded
    exclude = []

    height_cell = HeightCell(dimensions, heights, exclude)

    return height_cell

def saveCellAsImage(cell, image_name):
    """
    Save a HeightCell's height data to an image
    cell should be a valid HeightCell instance
    """

    def scaleBetween(x, in_min, in_max, out_min, out_max):
        return (x - in_min) * (out_max - out_min) / (in_max - in_min) + out_min

    # Scale point data so it goes 0-255
    # Excluded values are set to 0 (black)
    scaleTo255 = lambda x: scaleBetween(x, cell.min, cell.max, 0, 255) if x is not None else 0
    scaled_pixels = list(map(scaleTo255, cell.flattened))

    # Save image
    img = Image.new("L", (cell.xsize, cell.ysize))
    img.putdata(scaled_pixels)
    img.save(image_name)

def chooseSquare(base_dir):
    """
    Choose one of the 100x100km squares from the OS National Grid
    Prompts the user to enter the two-letter name of the chosen square
    """
    valid_square_names = sorted(os.listdir(base_dir))

    print("OS National Grid squares:")
    for a, b, c, d in zip(valid_square_names[::4], valid_square_names[1::4],
                          valid_square_names[2::4], valid_square_names[3::4]):
        print("{}, {}, {}, {}".format(a, b, c, d))

    square_name = input("Enter a square name, or multiple space-separated:\n")

    square_names = square_name.lower().split(" ")

    for name in square_names:
        if name not in valid_square_names:
            raise ValueError("\"{}\" not a valid square name".format(name))

    return square_names

def extractAscsFromSquare(base_dir, tmp_dir, square_name):
    """
    Extract all .asc files from the .zip files in the chosen square
    The files are extracted into tmp_dir

    Returns list of the file names
    """
    square_base_dir = os.path.join(base_dir, square_name)

    # Find out if this square is already extracted
    imported_asc_files = sorted(os.listdir(tmp_dir))
    is_selected_square = (f[:2].lower() == square_name for f in imported_asc_files)
    already_extracted = any(is_selected_square)

    # No need to extract any files - just return this square's ones
    if already_extracted:
        return filter(lambda f: f[:2].lower() == square_name,
                      imported_asc_files)

    # Otherwise get the files from the zip file
    file_names = []

    for item in os.listdir(square_base_dir):
        if not item.endswith(".zip"):
            continue    # Skip non-zip files

        item_path = os.path.join(square_base_dir, item)

        with zipfile.ZipFile(item_path, "r") as zip_ref:
            contents = zip_ref.namelist()
            asc_files = filter(lambda f: f.endswith(".asc"), contents)

            for asc_file in asc_files:
                zip_ref.extract(asc_file, tmp_dir)
                file_names.append(asc_file)

    return sorted(file_names)

def extractCellDataFromAscs(map_data_dir, asc_list):
    """
    Return HeightCell instances for all .asc files in asc_list
    The asc files should be in map_data_dir already
    """

    return [importAsc(os.path.join(map_data_dir, a)) for a in asc_list]

def getDimensions(height_cells):
    """
    Get dimensions needed for scaling
    Return dict:
        cell_size   = Cell size in metres
        cell_side   = Number of measurements (pixels) along one side of a cell
        cell_rows   = Number of rows of cells in the image
        img_width   = Width in pixels of the output image
        img_height  = Height in pixels of the output image
        min_x       = Minimum x coordinate (metres)
        min_y       = Minimum y coordinate (metres)
    """
    # Setup image grid parameters
    cell_size = 10000           # In metres
    measurement_interval = 50   # 50m per measurement
    cell_side = cell_size / measurement_interval
    cell_res = cell_side ** 2

    # Get image dimensions from real-life size
    x_corners = [c.xcorner for c in height_cells]
    y_corners = [c.ycorner for c in height_cells]
    max_x = max(x_corners) + cell_size
    max_y = max(y_corners) + cell_size
    min_x = min(x_corners)
    min_y = min(y_corners)

    ground_width = (max_x - min_x)
    ground_height = (max_y - min_y)
    cell_cols = ground_width // cell_size
    cell_rows = ground_width // cell_size
    img_width = int(ground_width * cell_side // cell_size)
    img_height = int(ground_height * cell_side // cell_size)

    return {"cell_size": cell_size,
            "cell_side": cell_side,
            "cell_rows": cell_rows,
            "img_width": img_width,
            "img_height": img_height,
            "min_x": min_x,
            "min_y": min_y,
            "max_y": max_y}

def scaleHeightData(heights):
    """
    Scale height data from its current range to 0-255 for image output
    """

    min_height = heights.min()

    # Shift any negative values up
    if min_height < 0:
        heights -= min_height

    # Scale heights to 255
    heights *= 255.0 / heights.max()
    return heights

def combineCells(height_cells):
    """
    Combine all height cells into one array and return it
    """

    dims = getDimensions(height_cells)

    # Use zero for default height (sea)
    heights_combined = np.zeros((dims["img_height"], dims["img_width"]))

    for cell in height_cells:
        # Get the start x and y indices of the cell
        cell_col = (cell.xcorner - dims["min_x"]) / dims["cell_size"]
        cell_row = (dims["max_y"] - cell.ycorner) / dims["cell_size"] - 1
        cell_start_x = int(cell_col * dims["cell_side"])
        cell_start_y = int(cell_row * dims["cell_side"])

        # Add actual height data to heights_combined
        for row, row_data in enumerate(cell.heights):
            for col, h in enumerate(row_data):
                # Get the pixel locations of this pixel
                px_col = cell_start_x + col
                px_row = cell_start_y + row

                heights_combined[px_row][px_col] = h

    return heights_combined

def makeImage(base_dir, map_data_dir, image_name, square_names, verbose=False):
    """
    Generate an image from the chosen square
    """

    if verbose:
        print("Importing map squares from .asc files")

    asc_files = []
    for square in square_names:
        # Add all .asc files from this square
        asc_files.extend(
            extractAscsFromSquare(base_dir, map_data_dir, square))

    # Import the cells as HeightCell objects
    height_cells = extractCellDataFromAscs(map_data_dir, asc_files)

    if verbose:
        print("Merging cells into one array")

    # Merge all HeightCells into one array
    heights = combineCells(height_cells)

    if verbose:
        print("Scaling to useful pixel brightness range")

    # Scale to 0-255
    heights = scaleHeightData(heights)

    if verbose:
        print("Saving image")

    # Ensure extension on image name
    if not image_name.endswith(".png"):
        image_name += ".png"

    # Save image
    img = Image.new("L", heights.shape[::-1])
    img.putdata(heights.flatten())
    img.save(image_name)

    # Remove map_data_dir so files aren't included in the next iteration
    #rmtree(map_data_dir)

    if verbose:
        print("Done!")

def interactiveMakeImage(base_dir, map_data_dir, image_name, verbose):
    """
    Interactively select grid squares and generate an image
    """

    square_names = chooseSquare(base_dir)

    image_name_new = input(
            "Image name (default=\"{}\"): ".format(image_name))

    if image_name_new != "":
        image_name = image_name_new

    makeImage(base_dir, map_data_dir, image_name, square_names, verbose)

if __name__ == "__main__":
    base_dir = os.path.join(".", "OS - terr50_gagg_gb", "data")
    map_data_dir = os.path.join(".", "map_data")
    image_name = "image"

    interactiveMakeImage(base_dir, map_data_dir, image_name, verbose=True)


import os
import argparse
import pandas as pd
from lxml import etree

NAMESPACE = 'http://openquake.org/xmlns/nrml/0.4'
GML_NAMESPACE = 'http://www.opengis.net/gml'
SERIALIZE_NS_MAP = {None: NAMESPACE, 'gml': GML_NAMESPACE}

def csv_to_xml(input_csv, output_xml):
    """
    Converts the CSV vulnerability model file to the NRML format
    """
    data = pd.io.parsers.read_csv(input_csv)
    grouped_by_set = data.groupby(['vuln_set_id','vuln_func_id'])
    vuln_model = {}
    for (vuln_set_id, vuln_func_id), group in grouped_by_set:
        if vuln_set_id not in vuln_model:
            vuln_model[vuln_set_id] = {}
            vuln_model[vuln_set_id]['asset_cat'] = group['asset_cat'].tolist()[0]
            vuln_model[vuln_set_id]['loss_cat'] = group['loss_cat'].tolist()[0]
            vuln_model[vuln_set_id]['imt'] = group['imt'].tolist()[0]
            vuln_model[vuln_set_id]['iml'] = group['iml'].tolist()
            vuln_model[vuln_set_id]['functions'] = []

        vuln_func = {}
        vuln_func['vuln_func_id'] = vuln_func_id
        vuln_func['distr'] = group['distr'].tolist()[0]
        vuln_func['mean_lr'] = group['mean_lr'].tolist()
        vuln_func['stddev_lr'] = group['stddev_lr'].tolist()

        vuln_model[vuln_set_id]['functions'].append(vuln_func)

    with open(output_xml, "w") as f:
        root = etree.Element('nrml', nsmap=SERIALIZE_NS_MAP)
        node_vm = etree.SubElement(root, "vulnerabilityModel")
        for vuln_set_id, vuln_func_set in vuln_model.iteritems():
            node_dvs = etree.SubElement(node_vm, "discreteVulnerabilitySet")
            node_dvs.set("vulnerabilitySetID", vuln_set_id)
            node_dvs.set("assetCategory", vuln_func_set['asset_cat'])
            node_dvs.set("lossCategory", vuln_func_set['loss_cat'])
            node_iml = etree.SubElement(node_dvs, "IML")
            node_iml.set("IMT", vuln_func_set['imt'])
            node_iml.text = " ".join(map(str, vuln_func_set['iml']))

            for vuln_func in vuln_func_set['functions']:
                node_dvf = etree.SubElement(node_dvs, "discreteVulnerability")
                node_dvf.set("vulnerabilityFunctionID", vuln_func['vuln_func_id'])
                node_dvf.set("probabilisticDistribution", vuln_func['distr'])
                node_lr = etree.SubElement(node_dvf, "lossRatio")
                node_lr.text = " ".join(map(str, vuln_func['mean_lr']))
                node_cv = etree.SubElement(node_dvf, "coefficientsVariation")
                node_cv.text = " ".join(map(str, vuln_func['stddev_lr']))

        f.write(etree.tostring(root, pretty_print=True, xml_declaration=True, encoding='UTF-8'))

def xml_to_csv (input_xml, output_csv):
    """
    Converts the XML vulnerability model file to the CSV format
    """
    print('This feature will be implemented in a future release.')

def set_up_arg_parser():
    """
    Can run as executable. To do so, set up the command line parser
    """

    description = ('Convert a Vulnerability Model from CSV to XML and '
                   'vice versa.\n\nTo convert from CSV to XML: '
                   '\npython vulnerability_model_converter.py '
                   '--input-csv-file PATH_TO_VULNERABILITY_MODEL_CSV_FILE '
                   '--output-xml-file PATH_TO_OUTPUT_XML_FILE'
                   '\n\nTo convert from XML to CSV type: '
                   '\npython vulnerability_model_converter.py '
                   '--input-xml-file PATH_TO_VULNERABILITY_MODEL_XML_FILE '
                   '--output-csv-file PATH_TO_OUTPUT_CSV_FILE')

    parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawTextHelpFormatter)
    flags = parser.add_argument_group('flag arguments')

    group_input = flags.add_mutually_exclusive_group(required=True)
    group_input.add_argument('--input-xml-file',
                       help='path to vulnerability model XML file',
                       default=None)
    group_input.add_argument('--input-csv-file',
                       help='path to vulnerability model CSV file',
                       default=None)

    group_output = flags.add_mutually_exclusive_group()
    group_output.add_argument('--output-xml-file',
                              help='path to output XML file',
                              default=None,
                              required=False)
    group_output.add_argument('--output-csv-file',
                              help='path to output CSV file',
                              default=None,
                              required=False)
    return parser

if __name__ == "__main__":

    parser = set_up_arg_parser()
    args = parser.parse_args()
    if args.input_csv_file:
        if args.output_xml_file:
            output_file = args.output_xml_file
        else:
            (filename, ext) = os.path.splitext(args.input_csv_file)
            output_file = filename + '.xml'
        csv_to_xml(args.input_csv_file, output_file)
    elif args.input_xml_file:
        if args.output_csv_file:
            output_file = args.output_csv_file
        else:
            (filename, ext) = os.path.splitext(args.input_xml_file)
            output_file = filename + '.csv'
        xml_to_csv(args.input_xml_file, output_file)
    else:
        parser.print_usage()


import argparse
import os
from utils import load_and_cache_examples, load_and_cache_unlabeled_examples, init_logger, load_tokenizer
from trainer import Trainer
import torch
import numpy as np
import random
import torch.nn as nn
from torch.utils.data import Subset
import json
import pickle
from transformers import AutoTokenizer
from eval import load_pred_data, save_data
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from collections import Counter

def set_seed(args):
    random.seed(args.train_seed)
    np.random.seed(args.train_seed)
    torch.manual_seed(args.train_seed)
    if args.n_gpu > 0  and torch.cuda.is_available():
        # print('yes')
        # assert 0
        torch.cuda.manual_seed_all(args.train_seed)
        torch.cuda.manual_seed(args.train_seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

def main(args):
    init_logger()
    set_seed(args)
    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer, use_fast=True)
    # tokenizer = load_tokenizer(args)

    train_dataset, num_labels, train_size  = load_and_cache_examples(args, tokenizer, mode="train")
    dev_dataset, num_labels,  dev_size = load_and_cache_examples(args, tokenizer, mode="dev")
    test_dataset, num_labels, test_size = load_and_cache_examples(args, tokenizer, mode="test")
    unlabeled_dataset, unlabeled_size = load_and_cache_unlabeled_examples(args, tokenizer, mode = 'unlabeled', train_size = train_size)
    contra_datasets = []

    print('number of labels:', num_labels)
    print('train_size:', train_size)
    print('dev_size:', dev_size)
    print('test_size:', test_size)
    print('unlabel_size:', unlabeled_size)

    trainer = Trainer(args, train_dataset=train_dataset, dev_dataset=dev_dataset,test_dataset=test_dataset, contra_datasets = contra_datasets, \
            unlabeled = unlabeled_dataset, num_labels = num_labels, data_size = train_size
            )
    if args.load_prev == 1:
        print(f"Load ckpt from {args.prev_ckpt}")
        trainer.load_model(path= args.prev_ckpt)

    trainer.init_model()
    if args.semi_method == 'ft':
        print("Begin Fine-Tuning!!")
        trainer.train(n_sample = len(train_dataset))
        train_pred, train_feat, train_label, unlabeled_pred, unlabeled_feat, unlabeled_label, unlabeled_pseudo = trainer.inference()
        save_data(train_pred, train_feat, train_label,  unlabeled_pred, unlabeled_feat, unlabeled_label, unlabeled_pseudo, mutual_info_bald = None, n_iter = args.round, dataset = args.task, ckpt = args.dr_model + f"_{args.dr_N}")
    else:
        print("Begin Self-Training!!")
        trainer.load_model(path= args.prev_ckpt)
        trainer.init_model()
        try:
            train_pred, train_feat, train_label, unlabeled_pred, unlabeled_feat, unlabeled_label, unlabeled_pseudo = load_pred_data(dataset = args.task, ckpt = args.dr_model, n_iter = args.round)
        except:
            train_pred, train_feat, train_label, unlabeled_pred, unlabeled_feat, unlabeled_label, unlabeled_pseudo = trainer.inference()
            save_data(train_pred, train_feat, train_label,  unlabeled_pred, unlabeled_feat, unlabeled_label, unlabeled_pseudo, mutual_info_bald = None, n_iter = args.round, dataset = args.task, ckpt = args.dr_model)
            train_pred, train_feat, train_label, unlabeled_pred, unlabeled_feat, unlabeled_label, unlabeled_pseudo = load_pred_data(dataset = args.task, ckpt = args.dr_model, n_iter = args.round)

        current_val = -np.max(unlabeled_pred, axis = -1)
        idx = np.argsort(current_val)
        cnt = Counter()
        labeled_idx = []
        for i in idx[:int(args.num_unlabeled)]:
            labeled_idx.append(i)
            cnt[np.argmax(unlabeled_pred[i])] += 1
        trainer.unlabeled = Subset(trainer.unlabeled, idx[:int(args.num_unlabeled)])
        trainer.selftrain()

    exit()

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument("--method", default='clean', type=str, help="which method to use")
    parser.add_argument("--gpu", default='0,1,2,3', type=str, help="which gpu to use")
    parser.add_argument("--n_gpu", default=1, type=int, help="which gpu to use")

    parser.add_argument("--seed", default=0, type=int, help="which seed to use")
    parser.add_argument("--train_seed", default=0, type=int, help="which seed to use")
    parser.add_argument("--task", default="agnews", type=str, help="The name of the task to train")

    parser.add_argument("--data_dir", default="../datasets", type=str,
                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
    parser.add_argument("--model_dir", default="./model", type=str, help="Path to model")
    parser.add_argument("--eval_dir", default="./eval", type=str, help="Evaluation script, result directory")
    parser.add_argument("--train_file", default="train.tsv", type=str, help="Train file")
    parser.add_argument("--dev_file", default="dev.tsv", type=str, help="dev file")
    parser.add_argument("--test_file", default="test.tsv", type=str, help="Test file")
    parser.add_argument("--unlabel_file", default="unlabeled.tsv", type=str, help="Test file")
    parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
    parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the test set.")

    parser.add_argument("--extra_dataset", default="", type=str, help="Whether to run extra eval on the test set.")

    parser.add_argument("--dev_labels", default=100, type=int, help="number of labels for dev set")
    parser.add_argument("--cache_dir", default="", type=str, help="Where do you want to store the pre-trained models downloaded from s3",)
    parser.add_argument("--output_dir", default=None, type=str, required=True, help="The output directory where the model predictions and checkpoints will be written.",)

    parser.add_argument('--logging_steps', type=int, default=20, help="Log every X updates steps.")
    parser.add_argument('--self_train_logging_steps', type=int, default=20, help="Log every X updates steps.")
    parser.add_argument('--save_steps', type=int, default=200, help="Save checkpoint every X updates steps.")
    parser.add_argument("--model_type", default="bert-base-uncased", type=str)
    parser.add_argument("--tokenizer", default="bert-base-uncased", type=str)

    parser.add_argument("--auto_load", default=1, type=int, help="Auto loading the model or not")
    parser.add_argument("--add_sep_token", action="store_true", help="Add [SEP] token at the end of the sentence")

    parser.add_argument("--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.")
    parser.add_argument("--max_steps", default=100, type=int, help="Training steps for initialization.")
    parser.add_argument("--weight_decay", default=1e-4, type=float, help="Weight decay if we apply some.")
    parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
    parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
    parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")
    parser.add_argument("--dropout_rate", default=0.1, type=float, help="Dropout for fully-connected layers")
    parser.add_argument("--batch_size", default=32, type=int, help="Batch size for training and evaluation.")
    parser.add_argument("--self_training_batch_size", default=32, type=int, help="Batch size for training and evaluation.")
    parser.add_argument("--eval_batch_size", default=256, type=int, help="Batch size for training and evaluation.")
    parser.add_argument("--learning_rate", default=1e-5, type=float, help="The initial learning rate for Adam.")
    parser.add_argument("--max_seq_len", default=128, type=int, help="The maximum total input sequence length after tokenization.")
    parser.add_argument("--max_seq_len_test", default=128, type=int, help="The maximum total input sequence length after tokenization.")

    parser.add_argument("--gradient_accumulation_steps", default=1, type=int, help="The maximum total input sequence length after tokenization.")
    parser.add_argument("--learning_rate_st", default=1e-5, type=float, help="The initial learning rate for self-training.")
    parser.add_argument('--self_training_max_step', type = int, default = 10000)
    parser.add_argument('--soft_label', type = int, default = 1)
    parser.add_argument('--self_training_update_period', type = int, default = 100, help = 'update period')
    parser.add_argument("--self_training_eps", default=0.6, type=float)
    parser.add_argument("--self_training_power", default=2, type=float)
    parser.add_argument("--self_training_weight", default=0.5, type=float)
    parser.add_argument("--gce_loss_q", default=0.5, type=float)

    parser.add_argument('--semi_method', type = str, default = 'ft')
    parser.add_argument('--k_cal', type = int, default = 5)

    parser.add_argument('--num_unlabeled', type = int, default = 100, help = 'number of unlabeled data.')

    parser.add_argument('--dr_model', type = str, default = 'arxiv_ckpt', help = 'the name for DR model.')
    parser.add_argument('--dr_N', type = int, default = 10, help = 'the number of retrieved documents.')

    parser.add_argument('--filter', type = int, default = 0, help = 'Whether use filter.')
    parser.add_argument('--template', type = int, default = 0, help = 'Prompt Template.')
    parser.add_argument('--round', type = int, default = 0, help = 'Prompt Template.')
    parser.add_argument('--load_prev', type = int, default = 0, help = 'whether load model from prev_round.')
    parser.add_argument('--prev_ckpt', type = str, default = '', help = 'dir of prev ckpt.')
    parser.add_argument('--train_full_labels', type = int, default = 0, help = 'whether train with full labels or not')

# semi_method
    args = parser.parse_args()
    if 'biobert' in args.model_type:
        args.model_name_or_path = 'dmis-lab/biobert-v1.1'
    elif 'scibert' in args.model_type:
        args.model_name_or_path = 'allenai/scibert_scivocab_uncased'
        args.tokenizer = 'allenai/scibert_scivocab_uncased'
    else:
        args.model_name_or_path = args.model_type
        args.tokenizer=args.model_type
    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
    print(os.environ['CUDA_VISIBLE_DEVICES'])
    main(args)


from django.shortcuts import get_object_or_404
from rest_framework.response import Response
from rest_framework.throttling import UserRateThrottle, AnonRateThrottle
from rest_framework.views import APIView

from plane.models import Plane, Ticket
from .serializer import TicketSerializer, PlaneSerializer

class PlaneAPIView(APIView):
    throttle_classes = [AnonRateThrottle]

    def get(self, request, *args, **kwargs):
        pk = self.kwargs.get('pk')
        if pk is not None:
            # Logic for retrieving a specific reservation by pk
            try:
                plane = Plane.objects.get(pk=pk)
                serializer = PlaneSerializer(plane)
                return Response(serializer.data)
            except Plane.DoesNotExist:
                return Response(status=404)  # Return a not found response if the reservation doesn't exist
        else:
            # Logic for retrieving all reservations
            planes = Plane.objects.all()
            serializer = PlaneSerializer(planes, many=True)
            return Response(serializer.data)

class TicketAPIView(APIView):
    throttle_classes = [UserRateThrottle]

    def get(self, request, *args, **kwargs):
        pk = self.kwargs.get('pk')
        if pk is not None:
            # Logic for retrieving a specific reservation by pk
            try:
                ticket = Ticket.objects.get(pk=pk)
                serializer = TicketSerializer(ticket)
                return Response(serializer.data)
            except Ticket.DoesNotExist:
                return Response(status=404)  # Return a not found response if the reservation doesn't exist
        else:
            # Logic for retrieving all reservations
            reservations = Ticket.objects.all()
            serializer = TicketSerializer(reservations, many=True)
            return Response(serializer.data)

    def post(self, request):
        plane_id = request.data.get('plane_id')
        total = request.data.get('total')

        plane = get_object_or_404(Plane, pk=plane_id)
        if not plane.capacity != 0:
            return Response({'error': 'The plane is full.'}, status=400)

        reservation = Ticket(
            user=request.my_user,
            plane=plane,
            total=total
        )
        reservation.save()
        if plane.capacity - total < 0:
            return Response({'error': 'request is more that plane capacity.'}, status=400)
        plane.capacity -= total
        plane.save()

        serializer = TicketSerializer(reservation)

        return Response(serializer.data)

    def __delete__(self, request, pk):
        try:
            reservation = Ticket.objects.get(pk=pk)

            plane = reservation.plane
            plane.capacity += reservation.total
            plane.save()

            reservation.delete()
            return Response(status=204)  # Return a success response if deletion is successful
        except Ticket.DoesNotExist:
            return Response(status=404)  # Return a not found response if the instance doesn't exist


import sys
from numbers import Real

from hypothesis import given

from shewchuk import Expansion
from tests.utils import skip_reference_counter_test
from . import strategies

@given(strategies.expansions)
def test_basic(expansion: Expansion) -> None:
    result = expansion.imag

    assert isinstance(result, int)

@given(strategies.expansions)
def test_zeroness(expansion: Expansion) -> None:
    result = expansion.imag

    assert result == 0

@skip_reference_counter_test
@given(strategies.expansions)
def test_reference_counter(expansion: Expansion) -> None:
    expansion_refcount_before = sys.getrefcount(expansion)

    result = expansion.imag

    expansion_refcount_after = sys.getrefcount(expansion)
    assert expansion_refcount_after == expansion_refcount_before


import cv2
import numpy as np
from tensorflow.keras.applications.mobilenet_v2 import preprocess_input
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.models import load_model

prototext_file = 'models/face_detector.prototxt'
weights_file = 'models/face_detector.caffemodel'
mask_model = 'models/mask_detector.model'
facenet = cv2.dnn.readNet(prototext_file, weights_file)
masknet = load_model(mask_model)
confidence = 0.5

def main(ctx, msg):
    image = cv2.imread('test1.jpg')
    (h, w) = image.shape[:2]
    blob = cv2.dnn.blobFromImage(image, 1.0, (300, 300), (104.0, 177.0, 123.0))
    facenet.setInput(blob)
    detections = facenet.forward()

    faces = []
    face_locations = []
    faces_without_mask = 0
    faces_with_mask = 0

    for i in range(0, detections.shape[2]):
        image_confidence = detections[0, 0, i, 2]

        if image_confidence > confidence:
            # compute the (x, y)-coordinates of the bounding box for the object
            box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
            (startX, startY, endX, endY) = box.astype('int')

            # ensure the bounding boxes fall within the dimensions of the frame
            (startX, startY) = (max(0, startX), max(0, startY))
            (endX, endY) = (min(w - 1, endX), min(h - 1, endY))

            # extract the face ROI, convert it from BGR to RGB channel ordering, resize it to 224x224, and preprocess it
            face = image[startY:endY, startX:endX]
            face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)
            face = cv2.resize(face, (224, 224))
            face = img_to_array(face)
            face = preprocess_input(face)

            # add the face and bounding boxes to their respective to result lists
            faces.append(face)
            face_locations.append((startX, startY, endX, endY))

    # only make a predictions if at least one face was detected
    if len(faces) > 0:
        # for faster inference we'll make batch predictions on *all*
        # faces at the same time rather than one-by-one predictions
        # in the above `for` loop
        faces = np.array(faces, dtype="float32")
        mask_predictions = masknet.predict(faces, batch_size=32)

    print(mask_predictions)
    print(type(mask_predictions))
    # draw boxes around the faces
    for (box, pred) in zip(face_locations, mask_predictions):
        (startX, startY, endX, endY) = box
        (mask, withoutMask) = pred
        print(pred)
        print(type(pred))

        # determine the class label and color we'll use to draw
        # the bounding box and text
        if mask > withoutMask:
            label = f'Mask: {mask:.2f}'
            faces_with_mask += 1
            color = (0, 255, 0)
        else:
            label = f'No Mask: {withoutMask:.2f}'
            faces_without_mask += 1
            color = (0, 0, 255)

        cv2.putText(image, label, (startX, startY - 10),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.75, color, 2)
        cv2.rectangle(image, (startX, startY), (endX, endY), color, 4)

    print('done')
    cv2.imshow('output', image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main('test', 'test')


#!/usr/bin/env python3

##The scholar crawler v 1.0
# It's as elegant as a drunk elephant

import argparse
from scholarly import scholarly
import re
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from tqdm import tqdm

## Going to use object orientation to keep publications data organized
## Not going to make the whole script object oriented though

class Paper:
    def __init__(self, pub):
        self.pub = pub
        self.url = None
        self.priority = 0
        self.text = None
        self.year = None
        self.domain = None
        self.period = None
        self.citations = None
        self.minim_citations = None

    def read(self):
        """Read contents of a web page and closes the browser"""

        #Open chrome silently in the background

        chrome_options = Options()
        chrome_options.add_argument("--headless")

        #Read webpage data

        wd = webdriver.Chrome(options=chrome_options)
        wd.get(self.url)
        text = BeautifulSoup(wd.page_source,"lxml").get_text()
        wd.close()
        self.text = text

    def get_url(self):
        """Get publication url from publication data"""
        self.url = self.pub["pub_url"]

    def get_year(self):
        """Get publication year from the copyright"""
        self.year = int(self.pub["bib"]["pub_year"])

    def in_period(self, period):
        """Filter Paper by time period"""
        if "-" in period:
            period = period.split("-")
        else:
            period = period.split()

        if self.year:
            self.period = ((self.year > int(period[0])) and (self.year < int(period[1])))

    def keywords_filter(self, filter_words):
        """Assign a score to publication based on the presence of keywords"""
        if self.text:
            for word in filter_words:
                if re.findall(word.lower(), self.text):
                    self.priority = int(self.priority) + 1

    def filter_domains(self, domains):
        """Filter Paper by domain"""
        for domain in domains.split():
            if re.findall(domain.lower(), self.url):
                self.domain = re.findall(domain.lower(), self.url)[0]

    def get_citations(self):
        """get the number of citations from scholarly data"""
        if self.pub["num_citations"]:
            self.citations = int(self.pub["num_citations"])

    def filter_citations(self, citations):
        """Return if the number of the citations is igual or grater than the minimum"""
        if self.citations:
            self.minim_citations = (self.citations >= int(citations))

    def norm_priority(self):
        """Makes sure publication score is within 0 to 10 range"""
        if self.priority is not None:
            self.priority = int(self.priority) % 10
        else:
            self.priority = 0

def get_first_n_pubs(query, n):
    """Create a list of dictionaries with the first n publications data"""
    ## Must be done with next() since query is an iterator and
    ## isn't subscriptable
    return [next(query) for i in tqdm(range(0, n))]

def get_arguments():
    """Return the query filter parameters and output name from comandline arguments"""

    parser = argparse.ArgumentParser()

    parser.add_argument("-q", "--query", help="The query you want to search")
    parser.add_argument("-kw", "--keywords", help="The key words of interest")
    parser.add_argument("-n", "--number", help="Maximum number of papers in the output")
    parser.add_argument("-p", "--period", help="The time period to filter for (optional)")
    parser.add_argument("-d", "--domains", help="Filter by domain name (optional)")
    parser.add_argument("-c", "--citations", help="number of citations(optional)")
    parser.add_argument("-o", "--output", help="Output file name")

    return parser.parse_args()

def make_url_score_output(lst):

    """Converts the publications urls and scores into a string that can be written in the output file"""

    string = ""

    for paper in lst:

        string += f"{paper.url},  {paper.priority}\n\n"

    string = "#url, #score\n" + string

    return string

def write_output(string, filename):
    """Writes data into a file"""
    with open(filename, "w") as f:
        f.write(string)

def main():

    args = get_arguments()

    query = args.query #the query string
    keywords = args.keywords # keywords to refine the search
    n = int(args.number) #maximum number of publications to search
    filename = args.output #output file name
    period = args.period #time period to filter
    domains = args.domains #domain names to filter
    citations = args.citations #Minimum nuymber of citations

    ## Init a query object from scholarly
    search = scholarly.search_pubs(query)

    ## Iterates getting the first n publications data

    print("\nSearching Google scholar ... \n")

    pubs = get_first_n_pubs(search, n)

    papers = [] ## Publications which were treated will be from now on regarded as papers

    for pub in tqdm(pubs):
        try:
             papers.append(Paper(pub)) ## Initialize a Paper type object for every publication
        except Exception as e:
            raise e
        finally:
            pass

    print("\nCrawling .... \n")

    ##Load Paper data into list of Paper objects
    ##Read the page text

    for paper in tqdm(papers):
        try:
            paper.get_url() ## Get the publication url for a Paper object
            paper.read() ## Read the webpage content

            if paper.text:
                paper.keywords_filter(keywords) ##Assign a score to the publication
                paper.norm_priority()
            else:
                papers.remove(paper)
        except Exception as e:
            raise e
        finally:
            pass

    if period: ## If flag -p was used

        print("\nFiltering by period ... \n")

        for paper in papers:
            try:
                paper.get_year() #get publication year
                paper.in_period(period) # check if the publication year is within the time interval
            except Exception as e:
                raise e
            finally:
                pass

        #Filter the publications that are in the time interval set by the user
        papers = list(filter(lambda x: x.period, papers))

    if domains: ## If Domains is not None filter publications by domains

        print(f"\nFiltering by Domain ... \n")

        for paper in tqdm(papers):
            try:
                paper.filter_domains(domains) ##Checks if the publication url is a domain specified by the user
            except Exception as e:
                raise e
            finally:
                pass
        ## Filters the Papers based by if their url domain is in the user suplied input
        papers = list(filter(lambda x: x.domain, papers))

    if citations: ## if -c flag was used

        print(f"\nFiltering by Citations ... \n")

        for paper in tqdm(papers):
            try:
                paper.get_citations() #Get citations from publication data
                paper.filter_citations(citations) ## Checks if publication has minimum number of citations
            except Exception as e:
                raise e
            finally:
                pass
        ## Filters the publications by if they have the minimum number of citations
        papers = list(filter(lambda x: x.minim_citations , papers))

    #Sort the papers by their score
    papers.sort(key=lambda x: x.priority, reverse=True)

    #Make data into a string that can be written to the output file
    string = make_url_score_output(papers)

    #writes data to an output file
    write_output(string, filename)

    print("\nDone!")

if __name__ == '__main__':
    main()


from datetime import datetime
import unittest

from psycopg2.errors import UniqueViolation

from tests.implementations import create_test_database, TweetPgsqlRepository, \
    TweetStatsRepository

class TestItemToInsertStatement(unittest.TestCase):

    def test_trouble_case(self):
        repo = TweetStatsRepository()
        item = {
            'tweet_id': 'Ugx7w7FYrLQILWq-TE14AaABAg',
            'collected_at': datetime(2021, 11, 29, 3, 13, 53, 809005),
            'num_likes': 0}
        sql, _ = repo._item_to_insert_statement(item, upsert=True)
        self.assertIn('SET num_likes = ', sql)

class TestPostgresRepository(unittest.TestCase):

    def test_get_conditions_and_values(self):
        kwargs = {
            'a': None,
            'b': 1,
            'c': 'three',
        }
        conditions, values = TweetPgsqlRepository._get_conditions_and_values(
            **kwargs)
        expected_conditions = 'b = %s, c = %s'
        self.assertEqual(expected_conditions, conditions)
        expected_values = [1, 'three']
        self.assertEqual(expected_values, values)

    def test_get_selector_returns_star_if_no_projection(self):
        kwargs = {}
        selector = TweetPgsqlRepository._get_selector(**kwargs)
        expected = '*'
        self.assertEqual(expected, selector)

    def test_get_selector_returns_star_if_projection_is_none(self):
        kwargs = {'projection': None}
        selector = TweetPgsqlRepository._get_selector(**kwargs)
        expected = '*'
        self.assertEqual(expected, selector)

    def test_get_selector_returns_projection_if_specified(self):
        kwargs = {'projection': ['attr1', 'attr2']}
        selector = TweetPgsqlRepository._get_selector(**kwargs)
        expected = 'attr1,attr2'
        self.assertEqual(expected, selector)

    def test_add_inserts_item(self):
        db_name = 'test_add_inserts_item'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet = {'label': None, 'tweet_id': 1, 'tweet': 'tweet1'}
        repo.add(tweet)
        item = repo.get(1)
        self.assertEqual(tweet, item)

    def test_add_ignores_duplicates_when_requested(self):
        db_name = 'test_add_ignores_duplicates_when_requested'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet = {'tweet_id': 1, 'tweet': 'tweet1'}
        repo.add(tweet)
        error = False
        try:
            repo.add(tweet, ignore_duplicates=True)
        except UniqueViolation:
            error = True
        self.assertFalse(error)

    def test_add_errors_duplicates_when_requested(self):
        db_name = 'test_add_errors_duplicates_when_requested'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet = {'tweet_id': 1, 'tweet': 'tweet1'}
        repo.add(tweet)
        error = False
        try:
            repo.add(tweet, ignore_duplicates=False)
        except UniqueViolation:
            error = True
        self.assertTrue(error)

    def test_add_many(self):
        db_name = 'test_add_many'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1'}
        tweet2 = {'tweet_id': 2, 'tweet': 'tweet2'}
        repo.add_many([tweet1, tweet2])
        self.assertTrue(repo.exists(1))
        self.assertTrue(repo.exists(2))

    def test_add_many_rolls_back_on_error(self):
        db_name = 'test_add_many_rolls_back_on_error'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1'}
        tweet2 = {'tweet_id': 1, 'tweet': 'tweet1'}
        try:
            repo.add_many([tweet1, tweet2])
        except UniqueViolation:
            pass
        self.assertFalse(repo.exists(1))

    def test_all_returns_all_items(self):
        db_name = 'test_all_returns_all_items'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'label': None, 'tweet_id': 1, 'tweet': 'tweet1'}
        tweet2 = {'label': None, 'tweet_id': 2, 'tweet': 'tweet2'}
        repo.add(tweet1)
        repo.add(tweet2)
        items = list(repo.all())
        expected = [tweet1, tweet2]
        self.assertEqual(expected, items)

    def test_count_returns_number_of_items_in_table(self):
        db_name = 'test_count_returns_number_of_items_in_table'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1'}
        tweet2 = {'tweet_id': 2, 'tweet': 'tweet2'}
        tweet3 = {'tweet_id': 3, 'tweet': 'tweet3'}
        repo.add(tweet1)
        repo.add(tweet2)
        repo.add(tweet3)
        count = repo.count()
        self.assertEqual(3, count)

    def test_delete_removes_items(self):
        db_name = 'test_delete_removes_items'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1'}
        repo.add(tweet1)
        self.assertTrue(repo.exists(1))
        repo.delete(dict(tweet_id=1))
        self.assertFalse(repo.exists(1))

    def test_delete_many(self):
        db_name = 'test_delete_many'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1'}
        tweet2 = {'tweet_id': 2, 'tweet': 'tweet2'}
        repo.add_many([tweet1, tweet2])
        self.assertTrue(repo.exists(1))
        self.assertTrue(repo.exists(2))
        repo.delete_many([{'tweet_id': 1}, {'tweet_id': 2}])
        self.assertFalse(repo.exists(1))
        self.assertFalse(repo.exists(2))

    def test_exists_returns_true_when_item_exists(self):
        db_name = 'test_exists_returns_true_when_item_exists'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1'}
        repo.add(tweet1)
        self.assertTrue(repo.exists(1))

    def test_exists_returns_false_when_item_does_not_exist(self):
        db_name = 'test_exists_returns_false_when_item_does_not_exist'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        self.assertFalse(repo.exists(1))

    def test_get_returns_none_when_item_does_not_exist(self):
        db_name = 'test_get_returns_none_when_item_does_not_exist'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        item = repo.get(1)
        self.assertIsNone(item)

    def test_get_returns_item_when_exists(self):
        db_name = 'test_get_returns_item_when_exists'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1'}
        repo.add(tweet1)
        item = repo.get(1)
        self.assertIsInstance(item, dict)
        self.assertIsNotNone(item)

    def test_search_returns_correct_items(self):
        db_name = 'test_search_returns_correct_items'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1'}
        tweet2 = {'tweet_id': 2, 'tweet': 'tweet1'}
        tweet3 = {'tweet_id': 3, 'tweet': 'tweet3'}
        repo.add(tweet1)
        repo.add(tweet2)
        repo.add(tweet3)
        items = list(repo.search(tweet='tweet1'))
        self.assertEqual(2, len(items))

    def test_update_single_item(self):
        db_name = 'test_update_single_item'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet = {'tweet_id': 1, 'tweet': 'tweet1', 'label': 'a'}
        repo.add(tweet)
        result = repo.get(1)
        self.assertEqual('a', result['label'])
        tweet['tweet'] = 'edited tweet'
        tweet['label'] = 'b'
        repo.update(tweet, ['tweet_id'], ['tweet', 'label'])
        result = repo.get(1)
        self.assertEqual('edited tweet', result['tweet'])
        self.assertEqual('b', result['label'])

    def test_update_many_items(self):
        db_name = 'test_update_many_items'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1', 'label': 'a'}
        tweet2 = {'tweet_id': 2, 'tweet': 'tweet2', 'label': 'a'}
        repo.add(tweet1)
        repo.add(tweet2)
        result = repo.get(1)
        self.assertEqual('a', result['label'])
        result = repo.get(2)
        self.assertEqual('a', result['label'])
        tweet1['label'] = 'b'
        tweet2['label'] = 'b'
        repo.update_many([tweet1, tweet2], ['tweet_id'], ['label'])
        result = repo.get(1)
        self.assertEqual('b', result['label'])
        result = repo.get(2)
        self.assertEqual('b', result['label'])

    def test_upsert_ok_when_not_exists(self):
        db_name = 'test_upsert_ok_when_not_exists'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet = {'tweet_id': 1, 'tweet': 'tweet1', 'label': 'a'}
        repo.upsert(tweet)
        result = repo.get(1)
        self.assertEqual(tweet, result)

    def test_upsert_ok_when_exists(self):
        db_name = 'test_upsert_ok_when_exists'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet = {'tweet_id': 1, 'tweet': 'tweet1', 'label': 'a'}
        repo.upsert(tweet)
        result = repo.get(1)
        self.assertEqual(tweet, result)
        error = False
        try:
            repo.upsert(tweet)
        except UniqueViolation:
            error = True
        self.assertFalse(error)
        result = repo.get(1)
        self.assertEqual(tweet, result)

    def test_upsert_many(self):
        db_name = 'test_upsert_many'
        create_test_database(db_name)
        repo = TweetPgsqlRepository(db_name=db_name)
        tweet1 = {'tweet_id': 1, 'tweet': 'tweet1', 'label': 'a'}
        repo.add(tweet1)

        tweet1['label'] = 'b'
        tweet2 = {'tweet_id': 2, 'tweet': 'tweet2', 'label': 'b'}
        repo.upsert_many([tweet1, tweet2])

        result = repo.get(1)
        self.assertEqual('b', result['label'])
        result = repo.get(2)
        self.assertEqual('b', result['label'])

    def test_upsert_with_two_primary_keys(self):
        db_name = 'test_upsert_with_two_primary_keys'
        create_test_database(db_name)
        repo = TweetStatsRepository(db_name=db_name)
        now = datetime.now()
        tweet_stats = {
            'tweet_id': 1,
            'collected_at': now,
            'num_likes': 1}
        repo.upsert(tweet_stats)
        result = repo.get(1, now)
        self.assertEqual(tweet_stats, result)


import datetime as dt
import json
from os import getenv
from typing import Callable

import requests
from flask import Flask, jsonify, request

# create your API token, and set it up in Postman collection as part of the Body section
API_TOKEN = getenv('API_TOKEN')
# you can get API keys for free here - https://www.weatherapi.com/my/
WEATHER_API_KEY = getenv('WEATHER_API_KEY')

app = Flask(__name__)

url_base_url = 'http://api.weatherapi.com/v1'

def get_current_weather(q: str, lang: str = None):
    """
    Return the current weather for the given parameters.

    `q` could be the following:
        * Latitude and Longitude (Decimal degree) e.g: q=48.8567,2.3508
        * city name e.g.: q=Paris

    :param q: Query parameter based on which data is sent back.
    :param lang: Optional. Returns 'condition:text' field in API in the desired language.
    :return: The current weather for the given parameters.
    """
    url_endpoint = 'current.json'
    url = f"{url_base_url}/{url_endpoint}"

    params = {
        'key': WEATHER_API_KEY,
        'q': q
    }
    if lang:
        params['lang'] = lang

    response = requests.request('GET', url, params=params)
    return json.loads(response.text)

def get_weather_forecast(q: str, days: int, lang: str = None):
    """
    Return the forecast weather for the given parameters.

    `q` could be the following:
        * Latitude and Longitude (Decimal degree) e.g: q=48.8567,2.3508
        * city name e.g.: q=Paris

    `days` parameter value ranges between 1 and 14.
        If no days parameter is provided then only today's weather is returned

    :param q: Query parameter based on which data is sent back.
    :param days: Number of days of forecast required.
    :param lang: Optional. Returns 'condition:text' field in API in the desired language.
    :return: The weather forecast for the given parameters.
    """
    url_endpoint = 'forecast.json'
    url = f"{url_base_url}/{url_endpoint}"

    params = {
        'key': WEATHER_API_KEY,
        'q': q,
        'days': days
    }
    if lang:
        params['lang'] = lang

    response = requests.request('GET', url, params=params)
    return json.loads(response.text)

def get_weather_history(q: str, dt: str, end_dt: str = None, lang: str = None):
    """
    Return the history weather for the given parameters.

    `q` could be the following:
        * Latitude and Longitude (Decimal degree) e.g: q=48.8567,2.3508
        * city name e.g.: q=Paris

    `dt` and `end_dt` should be on or after 1st Jan, 2010 in yyyy-MM-dd format (i.e. dt=2010-01-01)
    `end_dt` should be greater than 'dt' parameter
        and difference should not be more than 30 days between the two dates.
    If `end_dt` isn't provided, the history weather will be returned only for `dt` date.

    :param q: Query parameter based on which data is sent back.
    :param dt: Start date, from which history weather is returned
    :param end_dt: End date, to which history weather is returned
    :param lang: Optional. Returns 'condition:text' field in API in the desired language.
    :return: The current weather for the given parameters.
    """
    url_endpoint = 'history.json'
    url = f"{url_base_url}/{url_endpoint}"

    params = {
        'key': WEATHER_API_KEY,
        'q': q,
        'dt': dt
    }
    if end_dt:
        params['end_dt'] = end_dt
    if lang:
        params['lang'] = lang

    response = requests.request('GET', url, params=params)
    return json.loads(response.text)

class InvalidUsage(Exception):
    status_code = 400

    def __init__(self, message, status_code=None, payload=None):
        Exception.__init__(self)
        self.message = message
        if status_code is not None:
            self.status_code = status_code
        self.payload = payload

    @property
    def to_dict(self):
        rv = dict(self.payload or ())
        rv["message"] = self.message
        return rv

def authorize_and_validate_request(f: Callable):
    def wrapper():
        json_data = request.get_json()

        if json_data.get("token") is None:
            raise InvalidUsage("token is required", status_code=400)

        token = json_data.get("token")

        if token != API_TOKEN:
            raise InvalidUsage("wrong API token", status_code=403)

        if not json_data.get('weather'):
            raise InvalidUsage('weather params must be provided', status_code=400)

        return f()
    wrapper.__name__ = f.__name__
    return wrapper

def add_execution_time_params_to_response(f: Callable[[], dict]):
    def wrapper():
        start_dt = dt.datetime.now()
        response = f()
        end_dt = dt.datetime.now()
        execution_time_params = {
            "event_start_datetime": start_dt.isoformat(),
            "event_finished_datetime": end_dt.isoformat(),
            "event_duration": str(end_dt - start_dt),
        }
        response.update(execution_time_params)
        return response
    wrapper.__name__ = f.__name__
    return wrapper

@app.errorhandler(InvalidUsage)
def handle_invalid_usage(error):
    response = jsonify(error.to_dict)
    response.status_code = error.status_code
    return response

@app.route("/")
def home_page():
    return '

KMA L2: Python Saas. Weather

'

@app.route(
    '/content/api/v1/integration/weather/current',
    methods=["POST"],
)
@authorize_and_validate_request
@add_execution_time_params_to_response
def current_weather_endpoint():
    json_data: dict = request.get_json()

    weather_params = json_data.get('weather')
    weather = get_current_weather(weather_params.get('q'), weather_params.get('lang'))

    result = {
        "weather": weather
    }

    return result

@app.route(
    '/content/api/v1/integration/weather/forecast',
    methods=["POST"],
)
@authorize_and_validate_request
@add_execution_time_params_to_response
def weather_forecast_endpoint():
    json_data: dict = request.get_json()

    weather_params = json_data.get('weather')
    weather = get_weather_forecast(
        weather_params.get('q'), weather_params.get('days'),
        weather_params.get('lang')
    )

    result = {
        "weather": weather
    }

    return result

@app.route(
    '/content/api/v1/integration/weather/history',
    methods=["POST"],
)
@authorize_and_validate_request
@add_execution_time_params_to_response
def weather_history_endpoint():
    json_data: dict = request.get_json()

    weather_params: dict = json_data.get('weather')
    weather = get_weather_history(
        weather_params.get('q'),
        weather_params.get('dt'), weather_params.get('end_dt'),
        weather_params.get('lang')
    )

    result = {
        "weather": weather
    }

    return result


'''
Find Contours of a Image

OpenCV official documents:
https://docs.opencv.org/3.4/d4/d73/tutorial_py_contours_begin.html#:~:text=What%20are%20contours%3F,better%20accuracy%2C%20use%20binary%20images.
'''
import cv2 as cv
import numpy as np

img = cv.imread('Photos/alma.jpg')
cv.imshow('alma', img)

# convert to gray scale
img_gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)

# OpenCV documents suggest that first apply canny edge detection or thresholding before finding contours
canny = cv.Canny(img_gray, 125, 175)
cv.imshow('Canny Edges', canny)

# Find contours
contours, hierarchies = cv.findContours(canny, cv.RETR_LIST, cv.CHAIN_APPROX_SIMPLE)   # contours is a list; Each element is a list of coordinates

# Draw all the detected contours on a blank image
blank = np.zeros(img.shape, dtype='uint8')
blank = cv.drawContours(blank, contours, -1, (0, 255, 0), thickness=1)  # the thrid argument specify how many contours to draw
cv.imshow('Contours', blank)

cv.waitKey(0)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat May 13 18:50:37 2023

@author: jonas
"""

'''

IMPORTS

'''

# Standard library imports
import os
import sys
import glob
import ast
from pathlib import Path

# Related third party imports
import numpy as np
import pandas as pd
import h5py
import matplotlib.pyplot as plt
from matplotlib import pyplot
from scipy.signal import butter, lfilter, freqz, find_peaks, correlate, gaussian, filtfilt
from scipy import stats
from scipy import signal
from IPython.core.interactiveshell import InteractiveShell
from sklearn.mixture import GaussianMixture
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import quantities as pq
from quantities import ms, s, Hz
import pickle

# Local application/library specific imports
import neo
from neo.io import NeuroExplorerIO
import McsPy
import McsPy.McsData
import McsPy.McsCMOS
from McsPy import ureg, Q_
import elephant
from elephant.spike_train_generation import homogeneous_poisson_process, homogeneous_gamma_process
from elephant import neo_tools as nt
from elephant.spade import spade
from elephant.spade import pvalue_spectrum
from elephant.spade import concepts_mining
from elephant.spade import concept_output_to_patterns

InteractiveShell.ast_node_interactivity = 'all'

'''

FUNCTIONS

'''

def spikedic_to_neospiketrains(spikedic, recordinglength):

    spiketrains = []
    keylist_spiketrains = []
    for key in spikedic:
        key_array_sec=np.asarray(spikedic[key])

        # adjust the minimal amount of spikes per spiketrain
        if len(key_array_sec)>0:
            st = neo.SpikeTrain(list(key_array_sec), units='sec', t_stop=recordinglength)
            spiketrains.append(st)
            keylist_spiketrains.append(key)

    return spiketrains, keylist_spiketrains

def cut_reloaded_spikedic(spikedic, scale_factor_for_second, start=0, stop=60):

    cutted_dic = {}
    temp_list = []

    for key in spikedic:
        templist = []
        spikes = spikedic[key]
        for i in spikes:
            j = i*scale_factor_for_second # the ticks have already been applied in the burststarts
            if (j>=start) and (j<stop):
                templist.append(i)
        cutted_dic[key] = templist

    return cutted_dic

def remove_artefact_spikes(spikes, recording_length, tick, scale_factor_for_second, window=0.01):

    removing_times = []
    i = 0
    removed_spikes = {}
    kicked_out = {}
    while i < recording_length:
        activechannels = 0
        for key in spikes:
            for s in spikes[key]:
                if i <= s*tick*scale_factor_for_second < (i + window):
                    activechannels += 1
        if activechannels >= 100:
            removing_times.append(i)
        i += window

    print(removing_times)
    for key in spikes:
        new_spikes = []
        kicked_out_list = []

        s_list_save = list(spikes[key])
        s_list = list(spikes[key])
        for s in s_list_save:
            for time in removing_times:
                if time <= s*tick*scale_factor_for_second < (time + window*2):
                    kicked_out_list.append(s)
                    try:
                        s_list.remove(s)
                    except ValueError:
                        pass
                #else:
                 #   new_spikes.append(s)
        removed_spikes[key] = s_list
        kicked_out[key] = kicked_out_list

    return removed_spikes, kicked_out

def gaussian_smoothing(y, window_size=10, sigma=500):

    filt = signal.gaussian(window_size, sigma)

    return signal.convolve(y, filt, mode='same')

def convert_spiketimes_to_seconds(spike_dict, scale_factor, tick):
    """
    Convert spike times to seconds.

    Parameters:
    spike_dict (dict): Dictionary with channel labels as keys and lists of integer spike times as values.
    scale_factor (int): Scale factor to convert spike times to seconds.
    tick (int): Tick of the recordings.

    Returns:
    dict: A dictionary with channel labels as keys and lists of spike times in seconds as values.
    """

    # Convert spike_dict values to seconds
    spike_dict_seconds = {k: [tick_val * scale_factor * tick for tick_val in v] for k, v in spike_dict.items()}

    return spike_dict_seconds

def extract_spikes_in_lfp_periods(spike_dict_seconds, lfp_start_times, window=1.0):
    """
    Extract spikes that occur within a window period after each LFP start time, and adjust the spike times so
    that they appear as if the LFP periods occurred consecutively.

    Parameters:
    spike_dict_seconds (dict): Dictionary with channel labels as keys and lists of spike times in seconds as values.
    lfp_start_times (list): List of LFP start times.
    window (float, optional): Window period after each LFP start time. Default is 1.0.

    Returns:
    dict: A dictionary with channel labels as keys and lists of adjusted spike times within the LFP periods as values.
    """

    # Initialize an empty dictionary to store the spikes within the LFP periods
    spikes_in_lfp_periods = {k: [] for k in spike_dict_seconds.keys()}

    # Initialize a variable to keep track of the total time of the previous LFP periods
    total_previous_time = 0.0

    # Loop over the LFP start times
    for start_time in lfp_start_times:
        end_time = start_time + window

        # Loop over the channels
        for channel, spike_times in spike_dict_seconds.items():
            # Add spikes that fall within the window period after the LFP start time,
            # and adjust the spike times as if the LFP periods occurred consecutively
            spikes_in_lfp_periods[channel].extend([(t - start_time + total_previous_time) for t in spike_times if start_time <= t < end_time])

        # Add the current window period to the total time of the previous LFP periods
        total_previous_time += window

    return spikes_in_lfp_periods

'''

SCRIPT

'''

# for testing:
#/Users/jonas/Documents/DATA/SPADE_LFP_202305/SPADE_Trial/output/2022-04-06_cortex_div13_biometra_ID2203039CE_nodrug_spont_2

def main():

    #inputdirectory = input('Please enter the file directory: ')
    os.chdir(inputdirectory)
    filelist = glob.glob('*.pkl')

    print(filelist)
    df_list = []
    for i in filelist:
        filename = i
        filename = i.split('Dictionary_')[1].split('.pkl')[0]

        print('Working on ' + filename)
        print('this is element ' + str(filelist.index(i)) + ' of ' + str(len(filelist)))

        #create the outputdirectory
        mainoutputdirectory = os.path.join(inputdirectory, 'output_spade')
        outputdirectory = os.path.join(mainoutputdirectory, filename)
        try:
            Path(outputdirectory).mkdir(parents=True, exist_ok=False)
        except FileExistsError:
            pass

        '''
        all basic parameters are imported
        '''
        MAIN_RECORDING_DICTIONARY = pickle.load(open('MAIN_RECORDING_Dictionary_'+filename+'.pkl', "rb"))
        spikedic_MAD = MAIN_RECORDING_DICTIONARY['spikedic_MAD']
        tick = MAIN_RECORDING_DICTIONARY['Infos_Recording']['tick']
        timelengthrecording_s = MAIN_RECORDING_DICTIONARY['Infos_Recording']['timelengthrecording_s']
        info_dics_subrecordings = MAIN_RECORDING_DICTIONARY['Infos_Recording']['info_dics_subrecordings']
        Infos_Recording = MAIN_RECORDING_DICTIONARY['Infos_Recording']
        key_timepoint = list(info_dics_subrecordings.keys())[0]
        first_recording_timepoint = info_dics_subrecordings[key_timepoint]['first_recording_timepoint']
        subrec_infos = info_dics_subrecordings[key_timepoint]
        scale_factor_for_second = MAIN_RECORDING_DICTIONARY['Infos_Recording']['scale_factor_for_second']
        relevant_factor = timelengthrecording_s*0.05
        fr_dic = MAIN_RECORDING_DICTIONARY['fr_dic']

        '''
        removal of artefacts
        '''

        spikedic_MAD, kicked_out = remove_artefact_spikes(spikes=spikedic_MAD,
                                                          recording_length=timelengthrecording_s,
                                                          tick=tick,
                                                          scale_factor_for_second=scale_factor_for_second)

        '''
        remove

        '''
        active_channels = 0
        spikedic_seconds = {}
        for key in spikedic_MAD:
            sec_array = np.asarray(spikedic_MAD[key])*tick*scale_factor_for_second
            spikedic_seconds[key]=sec_array
            active_channels += 1

        spikearray_seconds = np.asarray(list(spikedic_seconds.values()))

        '''
        get one-dimensional array of every detected spike
        '''
        scale_factor_for_milisecond = 1e-03
        full_spike_list = []
        full_spike_list_seconds = []
        for key in spikedic_MAD:
            # without the relevant factor --every channel will be plotted
            x = list(np.asarray(spikedic_MAD[key])*scale_factor_for_milisecond*tick)
            full_spike_list = full_spike_list + x

            xs = list(np.asarray(spikedic_MAD[key])*scale_factor_for_second*tick)
            full_spike_list_seconds = full_spike_list_seconds + xs
        full_spikes = sorted(full_spike_list)
        full_spikes_seconds = sorted(full_spike_list_seconds)

        mean_fr_whole_recording = np.around((len(full_spikes_seconds) / timelengthrecording_s), 3)

        #define bins
        binsize = 0.005 #seconds
        #bins= np.arange(0, timelengthrecording_s+binsize, binsize)

        #trial of population burst plot as inspired by Andrea Corna
        bins = int(timelengthrecording_s / binsize)+1

        firing_rate_histogram = np.histogram(full_spikes_seconds, bins=bins)
        firing_rate = firing_rate_histogram[0]*200 #conversion to hertz
        #firing_rate = firing_rate_histogram[0]

        # using firing rate histogram already conversed to hertz
        N = int(1/binsize) # für eine Sekunde, das Sliding window, also letztlich number of bins
        plot_N = int(0.01/binsize)
        # gaussian smmothing fo the firing rate and moving average
        fr_gau = gaussian_smoothing(firing_rate)
        plot_fr_gau = gaussian_smoothing(firing_rate)
        ma_fr_gau = np.convolve(fr_gau, np.ones(N)/N, mode='full')
        plotting_ma_fr_gau = np.convolve(fr_gau, np.ones(plot_N)/plot_N, mode='full')

        # we look for the mean of the MA as threshold
        # we arrange this mean in an array for plotting
        mean_ma_fr_gau = np.mean(ma_fr_gau)
        std_ma_fr_gau = np.std(ma_fr_gau)
        network_burst_threshold = mean_ma_fr_gau
        shape_for_threshold = np.shape(ma_fr_gau)
        network_burst_threshold_array = np.full(shape_for_threshold, network_burst_threshold)

        '''
        PLOTTING OF THE corrected Rasterplot
        '''

        fig = plt.figure(figsize = (10,6))
        gs = fig.add_gridspec(2, hspace = 0, height_ratios=[2,5])
        axs = gs.subplots(sharex=False, sharey=False)
        axs[0].plot(ma_fr_gau, color= 'black', lw= 0.5)
        axs[0].set_ylabel('Firing Rate [Hz]')
        axs[0].set_ylim(0, 5000)
        axs[1].eventplot(spikearray_seconds, color = 'black', linewidths = 0.5,
                         linelengths = 1, colors = 'black')
        axs[1].set_ylabel('Relevant Channels')
        fig.suptitle(filename)

        '''
        remove until here

        '''
        lfp_list = [4.56, 73.9, 99.36, 147.26, 167.29, 179.9, 194.14, 217.1, 238.38, 254.1]

        spikedic_in_seconds = convert_spiketimes_to_seconds(spike_dict=spikedic_MAD,
                                                            scale_factor=scale_factor_for_second,
                                                            tick=tick)

        spikes_in_lfp_periods = extract_spikes_in_lfp_periods(spike_dict_seconds=spikedic_in_seconds,
                                                              lfp_start_times = lfp_list,
                                                              window=1.0)

        # add the parameters for SPADE
        bin_size = 5 * pq.ms # time resolution to discretize the spiketrains
        winlen = 10 # maximal pattern length in bins (i.e., sliding window)
        dither = 20 * pq.ms
        spectrum = '#'
        alpha = 0.05
        stat='fdr_bh'

        # get the recordinglenght needed to create the neo spikearray

        spikes = []
        for key in spikes_in_lfp_periods:
            spikes.append(spikes_in_lfp_periods[key])
            spikearray = np.sort(np.concatenate(spikes, axis = 0))
            spikelist = list(spikearray)

        recordinglength = round(spikearray[-1]) + 1

        # create the neo spiketrain
        spiketrains, keylist_spiketrains = spikedic_to_neospiketrains(spikedic=spikes_in_lfp_periods,
                                                                      recordinglength=recordinglength)

        result_spade = spade(spiketrains, bin_size=bin_size, winlen=winlen, n_surr=500, min_occ=4, min_spikes=2, min_neu=2, stat_corr=stat,
                       spectrum=spectrum, alpha=alpha)