# -*- coding: utf-8 -*-
# @File    : ResNet50.py
# @Author  : Hua Guo
# @Time    : 2020/3/8 上午8:26
# @Disc    :
from typing import List
import tensorflow as tf
from tensorflow.keras import layers, Model

class ConvBNActivation(Model):
    """
    conv + batch nomalization + relu
    """
    def __init__(self, filter: int, kernel_size: List[int], padding: str, activation: str, stride=None) -> None:
        super(ConvBNActivation, self).__init__()
        if stride is None:
            stride = [1, 1]
        self.conv = layers.Conv2D(
            filters=filter,
            kernel_size=kernel_size,
            padding=padding,
            strides=stride
        )
        self.bn = layers.BatchNormalization()
        self.activation = layers.Activation(activation)

    def call(self, inputs, training=None, mask=None):
        x = self.conv(inputs)
        x = self.bn(x)
        x = self.activation(x)
        return x

class IdentityBlock(Model):
    """S
    In the case where the input activation has the same dimension as the output activation
    Structures are as follow:
        3 conv in the main path
        1 identity in the shortcut path
    """
    def __init__(self, kernel_size: int, filters: List[int]) -> None:
        super(IdentityBlock, self).__init__()
        f1, f2, f3 = filters

        self.con_bn_ac1 = ConvBNActivation(filter=f1, kernel_size=[1, 1], padding="valid", activation="relu")
        self.con_bn_ac2 = ConvBNActivation(filter=f2, kernel_size=[kernel_size, kernel_size], padding="same", activation="relu")
        self.conv = layers.Conv2D(filters=f3, kernel_size=[1, 1], padding="same")
        self.bn = layers.BatchNormalization()
        self.activation = layers.Activation("relu")

    def call(self, inputs, training=None, mask=None):
        x = self.con_bn_ac1(inputs)
        for l in [
            self.con_bn_ac2,
            self.conv,
            self.bn
        ]:
            x = l(x)
        x = tf.math.add(x, inputs)
        return self.activation(x)

class ConvBlock(Model):
    """
    In the case where the dimension of the the input activation is different from that of the output activation
    Structures are as follow:
        3 conv in the main path
        1 conv in the shortcut path
    """
    def __init__(self, mid_kernel_size: int, shortcut_stride: int, filters: List[int]):
        super(ConvBlock, self).__init__()
        f1, f2, f3 = filters
        self.con_bn_ac1 = ConvBNActivation(filter=f1, kernel_size=[1, 1], padding="valid", activation="relu",
                                           stride=[shortcut_stride, shortcut_stride])
        self.con_bn_ac2 = ConvBNActivation(filter=f2, kernel_size=[mid_kernel_size, mid_kernel_size],
                                           padding="same", activation="relu")
        self.conv = layers.Conv2D(filters=f3, kernel_size=[1, 1], padding="same")
        self.bn = layers.BatchNormalization()
        self.activation = layers.Activation("relu")
        self.conv_shortcut = layers.Conv2D(filters=f3, kernel_size=[1, 1], padding="valid",
                                           strides=[shortcut_stride, shortcut_stride])
        self.bn_shortcut = layers.BatchNormalization()

    def call(self, inputs, training=None, mask=None):
        # shortcut branch
        x_short_cut = self.conv_shortcut(inputs)
        x_short_cut = self.bn_shortcut(x_short_cut)
        # main path
        x = self.con_bn_ac1(inputs)
        for l in [
            self.con_bn_ac2,
            self.conv,
            self.bn
        ]:
            x = l(x)
        x = tf.math.add(x, x_short_cut)
        return self.activation(x)

class ResNet50(Model):
    """
    Residual NN with 50 layers
    Structures stages are as follow:
        0. zero padding
        1. conv + bn + Relu + max pooling
        2. Convblock + IdentityBlock*2
        3. Convblock + IdentityBlock*2
        4, Convblock + IdentityBlock*5
        5, Convblock + IdentityBlock*2
        6. avg pooling + flatten + fc
    """
    def __init__(self, class_num: int) -> None:
        super(ResNet50, self).__init__()
        self.class_num = class_num
        # stage 1
        self.con_bn_ac = ConvBNActivation(
            filter=64,
            kernel_size=[7, 7],
            activation="relu",
            padding="valid"
        )
        self.max_pooling = layers.MaxPool2D((3, 3), strides=(2, 2))
        # stage 2
        self.conv_block1 = ConvBlock(
            mid_kernel_size=3,
            filters=[64, 64, 256],
            shortcut_stride=1
        )
        self.identity_block1 = IdentityBlock(
            kernel_size=3,
            filters=[64, 64, 256]
        )
        self.identity_block2 = IdentityBlock(
            kernel_size=3,
            filters=[64, 64, 256]
        )
        # stage 3
        self.conv_block2 = ConvBlock(
            mid_kernel_size=3,
            filters=[128, 128, 512],
            shortcut_stride=2
        )
        self.identity_block3 = IdentityBlock(
            kernel_size=3,
            filters=[128, 128, 512]
        )
        self.identity_block4 = IdentityBlock(
            kernel_size=3,
            filters=[128, 128, 512]
        )
        self.identity_block5 = IdentityBlock(
            kernel_size=3,
            filters=[128, 128, 512]
        )
        # stage 4
        self.conv_block3 = ConvBlock(
            mid_kernel_size=3,
            filters=[256, 256, 1024],
            shortcut_stride=2
        )
        self.identity_block6 = IdentityBlock(
            kernel_size=3,
            filters=[256, 256, 1024]
        )
        self.identity_block7 = IdentityBlock(
            kernel_size=3,
            filters=[256, 256, 1024]
        )
        self.identity_block8 = IdentityBlock(
            kernel_size=3,
            filters=[256, 256, 1024]
        )
        self.identity_block9 = IdentityBlock(
            kernel_size=3,
            filters=[256, 256, 1024]
        )
        self.identity_block10 = IdentityBlock(
            kernel_size=3,
            filters=[256, 256, 1024]
        )
        # stage 5
        self.conv_block4 = ConvBlock(
            mid_kernel_size=3,
            filters=[512, 512, 2048],
            shortcut_stride=2
        )
        self.identity_block11 = IdentityBlock(
            kernel_size=3,
            filters=[512, 512, 2048]
        )
        self.identity_block12 = IdentityBlock(
            kernel_size=3,
            filters=[512, 512, 2048]
        )

        self.avg_pooling = layers.AveragePooling2D(
            pool_size=(2, 2),
            padding="same"
        )
        self.flat = layers.Flatten()
        self.dese = layers.Dense(
            class_num,
            "softmax"
        )

    def call(self, inputs, training=None, mask=None):
        x = layers.ZeroPadding2D(padding=[3, 3])(inputs)
        for block in [
            # stage 1
            self.con_bn_ac,
            self.max_pooling,
            # stage 2
            self.conv_block1,
            self.identity_block1,
            self.identity_block2,
            # stage 3
            self.conv_block2,
            self.identity_block3,
            self.identity_block4,
            self.identity_block5,
            # stage 4
            self.conv_block3,
            self.identity_block6,
            self.identity_block7,
            self.identity_block8,
            self.identity_block9,
            self.identity_block10,
            # stage 5
            self.conv_block4,
            self.identity_block11,
            self.identity_block12,
            self.avg_pooling,
            self.flat,
            self.dese

        ]:
            x = block(x)
        return x

if __name__ == "__main__":
    con_bn_activation = ConvBNActivation(
        filter=10,
        kernel_size=[3, 3],
        padding="valid",
        activation="relu"
    )
    con_bn_activation(tf.ones(shape=(1, 32, 32, 3)))
    print(con_bn_activation.summary())
    # identity = IdentityBlock()
    identity_block = IdentityBlock(
        kernel_size=3,
        filters=[3, 10, 3]
    )
    identity_block(tf.ones(shape=(1, 32, 32, 3)))
    print(identity_block.summary())
    conv_block = ConvBlock(
        mid_kernel_size=3,
        filters=[3, 10, 5],
        shortcut_stride=2
    )
    conv_block(tf.ones(shape=(1, 32, 32, 10)))
    print(conv_block.summary())

    resnet50 = ResNet50(class_num=238)
    resnet50(tf.ones(shape=(1, 32, 32, 3)))
    print(resnet50.summary())


from setuptools import setup, find_packages

VERSION = '0.1.2.0'

setup(
    name="lhub_cli",
    version=VERSION,
    author="Chad Roberts",
    author_email="chad@logichub.com",
    description="LogicHub CLI",
    long_description="A Python package for interacting with LogicHub via shell commands",
    packages=find_packages(),
    install_requires=[
        "lhub ~= 0.2.6",
        "configobj >= 5.0.6",
        "dataclasses_json >= 0.5.6",
        "rsa >= 4.8",
        "requests",
        "tabulate",
        "colorama",
        "structlog",
        "progressbar2",  # Only used in sample scripts for now, and imported not as "progressbar2" but just "progressbar"
    ],
    keywords=["python", "lhub", "LogicHub", "api", "security", "cli"],

    url="https://github.com/deathbywedgie/lhub_cli",
    project_urls={
        "Bug Tracker": "https://github.com/deathbywedgie/lhub_cli/issues",
    },

    # https://pypi.org/classifiers/
    classifiers=[
        "Development Status :: 1 - Planning",
        "Intended Audience :: End Users/Desktop",
        "Intended Audience :: System Administrators",
        "Intended Audience :: Information Technology",
        "Programming Language :: Python :: 3.10",
        "Natural Language :: English",
    ]
)


"""
REST

HTTPメソッド クライアントが行いたい処理をサーバーに伝える

GET     データの参照
POST    データの新規登録
PUT     データの更新
DELETE  データの削除
"""

import urllib.request
import json

payload = {'key1': 'value1', 'key2': 'value2'}

# GET
# url = 'http://httpbin.org/get' + '?' + urllib.parse.urlencode(payload)
# print(url)

# with urllib.request.urlopen(url) as f:
    # decode することで人の目できれいに見えるようにできる
    # Json形式で返ってくる
    # print(f.read().decode('utf-8'))
    # Python の辞書型で返す場合
    # 返ってきたものを Python で扱える
    # r = json.loads(f.read().decode('utf-8'))
    # print(type(r))

# POST
# Request のオブジェクトを渡す
payload = json.dumps(payload).encode('utf-8')
# req = urllib.request.Request(
#     'http://httpbin.org/post', data=payload, method='POST'
# )
# with urllib.request.urlopen(req) as f:
#     print(json.loads(f.read().decode('utf-8')))

# PUT
req = urllib.request.Request(
    'http://httpbin.org/put', data=payload, method='PUT'
)
with urllib.request.urlopen(req) as f:
    print(json.loads(f.read().decode('utf-8')))

req = urllib.request.Request(
    'http://httpbin.org/delete', data=payload, method='DELETE'
)
with urllib.request.urlopen(req) as f:
    print(json.loads(f.read().decode('utf-8')))


import allure
from allure_commons.types import AttachmentType
from selenium.webdriver.common.by import By
from pageobjects.base_page import BasePage

class MainPage(BasePage):

    SELECTED_COUNTRY = (By.CSS_SELECTOR, ".countries-dropdown.dropdown.small > .select")
    COUNTRIES_IN_THE_LIST = (
        By.CSS_SELECTOR,
        ".options > .options-container > .option > span > span",
    )
    MENU_WOMEN = (
        By.XPATH,
        "//section[@class='header-main-category-menu']/ul[@class='list']/li[1]",
    )
    MARKETING_APPROVALS = (By.CSS_SELECTOR, "div#marketing-approvals > .modal")
    MARKETING_APPROVALS_CLOSE = (By.CSS_SELECTOR, ".close-icon-wrapper .svg")

    def __init__(self, driver):
        super().__init__(driver)
        # self.driver = driver

    @allure.step("Check the language set")
    def check_language_set(self, language: str):
        """
        Check if correct language was set, if not select it.
        :param language: Country code e.g. PL
        """
        set_language = self.get_text_of(self.SELECTED_COUNTRY)
        if set_language is not language:
            self.click_on(self.SELECTED_COUNTRY)
            self.select_by_visible_text(language)

        allure.attach(
            self.driver.get_screenshot_as_png(),
            name="check_language_set",
            attachment_type=AttachmentType.PNG,
        )

    @allure.step("Click on women in top menu")
    def click_on_women_in_top_menu(self):
        """Go to women items."""
        ff = (By.CSS_SELECTOR, ".header-main-category-menu")
        self.element_is_visible(ff)
        self.element_is_visible(self.MENU_WOMEN)
        self.click_on(self.MENU_WOMEN)

        allure.attach(
            self.driver.get_screenshot_as_png(),
            name="click_on_women_in_top_menu",
            attachment_type=AttachmentType.PNG,
        )

    @allure.step("Close modal message if exist")
    def close_modal_message_if_exists(self):
        if self.check_if_exists(locator=self.MARKETING_APPROVALS, time_in_seconds=10):
            self.click_on(self.MARKETING_APPROVALS_CLOSE)
        else:
            pass
        allure.attach(
            self.driver.get_screenshot_as_png(),
            name="close_modal_message_if_exists",
            attachment_type=AttachmentType.PNG,
        )


from typing import Optional, Tuple, Union
import random
import numpy as np
from scipy.ndimage import median_filter
from ..._transform import Transformer

class MedianFilter(Transformer):
    def __init__(
        self,
        filter_size: Union[int, Tuple[int, int]]=(3, 5),
    ) -> None:
        self.filter_size = filter_size

    def augument_median_filter(
        self,
        inp: np.ndarray
    ) -> np.ndarray:

        for b in range(inp.shape[0]):
            for c in range(inp.shape[3]):
                if isinstance(self.filter_size, int):
                    filter_size = self.filter_size
                else:
                    filter_size = np.random.randint(*self.filter_size)
                inp[b, :, :, c] = median_filter(inp[b, :, :, c], filter_size)
        return inp

    def __call__(
        self,
        inp: np.ndarray,
        mask: np.ndarray,
        scale: Optional[float]=None
    ) -> Union[np.ndarray, np.ndarray]:

        inp = self.augument_median_filter(inp)
        return inp, mask

class RandomMedianFilter(Transformer):
    def __init__(self) -> None:
        super().__init__()
        self.median_filter_fn = [MedianFilter(), None]

    def __call__(
        self,
        inp: np.ndarray,
        mask: np.ndarray
    ) -> Union[np.ndarray, np.ndarray]:

        median_filter_fn = random.choice(self.median_filter_fn)
        if median_filter_fn is None:
            return inp, mask
        else:
            return median_filter_fn(inp, mask)


from Appconfig import Appconfig
import re
import os
import xml.etree.cElementTree as ET
from PyQt5 import QtWidgets

class AutoSchematic(QtWidgets.QWidget):

    def __init__(self, parent, modelname):
        QtWidgets.QWidget.__init__(self)
        self.parent = parent
        self.modelname = modelname.split('.')[0]
        self.template = Appconfig.kicad_sym_template.copy()
        self.xml_loc = Appconfig.xml_loc
        self.lib_loc = Appconfig.lib_loc
        if os.name == 'nt':
            eSim_src = Appconfig.src_home
            inst_dir = eSim_src.replace('\\eSim', '')
            self.kicad_nghdl_sym = \
                inst_dir + '/KiCad/share/kicad/symbols/eSim_Nghdl.kicad_sym'
        else:
            self.kicad_nghdl_sym = \
                '/usr/share/kicad/symbols/eSim_Nghdl.kicad_sym'
        self.parser = Appconfig.parser_nghdl

    def createKicadSymbol(self):
        xmlFound = None
        for root, dirs, files in os.walk(self.xml_loc):
            if (str(self.modelname) + '.xml') in files:
                xmlFound = root
                print(xmlFound)
        if xmlFound is None:
            self.getPortInformation()
            self.createXML()
            self.createSym()
        elif (xmlFound == os.path.join(self.xml_loc, 'Nghdl')):
            print('Library already exists...')
            ret = QtWidgets.QMessageBox.warning(
                self.parent, "Warning", '''Library files for this model''' +
                ''' already exist. Do you want to overwrite it?

                If yes press ok, else cancel it and ''' +
                '''change the name of your vhdl file.''',
                QtWidgets.QMessageBox.Ok, QtWidgets.QMessageBox.Cancel
            )
            if ret == QtWidgets.QMessageBox.Ok:
                print("Overwriting existing libraries")
                self.getPortInformation()
                self.createXML()
                self.removeOldLibrary()     # Removes the existng library
                self.createSym()
            else:
                print("Exiting Nghdl")
                quit()
        else:
            print('Pre existing library...')
            ret = QtWidgets.QMessageBox.critical(
                self.parent, "Error", '''A standard library already ''' +
                '''exists with this name.
Please change the ''' +
                '''name of your vhdl file and upload it again.''',
                QtWidgets.QMessageBox.Ok
            )

            # quit()

    def getPortInformation(self):
        '''
            getting the port information here
        '''
        portInformation = PortInfo(self)
        portInformation.getPortInfo()
        self.portInfo = portInformation.bit_list
        self.input_length = portInformation.input_len

    def createXML(self):
        '''
            creating the XML files in eSim /library/modelParamXML/Nghdl
        '''
        cwd = os.getcwd()
        xmlDestination = os.path.join(self.xml_loc, 'Nghdl')
        self.splitText = ""
        for bit in self.portInfo[:-1]:
            self.splitText += bit + "-V:"
        self.splitText += self.portInfo[-1] + "-V"

        print("changing directory to ", xmlDestination)
        os.chdir(xmlDestination)

        root = ET.Element("model")
        ET.SubElement(root, "name").text = self.modelname
        ET.SubElement(root, "type").text = "Nghdl"
        ET.SubElement(root, "node_number").text = str(len(self.portInfo))
        ET.SubElement(root, "title").text = (
                            "Add parameters for " + str(self.modelname))
        ET.SubElement(root, "split").text = self.splitText
        param = ET.SubElement(root, "param")
        ET.SubElement(param, "rise_delay", default="1.0e-9").text = (
                                    "Enter Rise Delay (default=1.0e-9)")
        ET.SubElement(param, "fall_delay", default="1.0e-9").text = (
                                    "Enter Fall Delay (default=1.0e-9)")
        ET.SubElement(param, "input_load", default="1.0e-12").text = (
                                    "Enter Input Load (default=1.0e-12)")
        ET.SubElement(param, "instance_id", default="1").text = (
                                    "Enter Instance ID (Between 0-99)")
        tree = ET.ElementTree(root)
        tree.write(str(self.modelname) + '.xml')
        print("Leaving the directory ", xmlDestination)
        os.chdir(cwd)

    def findBlockSize(self):
        '''
            Calculates the maximum between input and output ports
        '''
        ind = self.input_length
        return max(
            self.char_sum(self.portInfo[:ind]),
            self.char_sum(self.portInfo[ind:])
        )

    def char_sum(self, ls):
        return sum([int(x) for x in ls])

    def removeOldLibrary(self):
        '''
            removing the old library
        '''
        cwd = os.getcwd()
        os.chdir(self.lib_loc)
        print("Changing directory to ", self.lib_loc)
        sym_file = open(self.kicad_nghdl_sym)
        lines = sym_file.readlines()
        lines = lines[0:-2]
        sym_file.close()

        output = []
        line_reading_flag = False

        for line in lines:
            if line.startswith("(symbol"):      # Eeschema template start
                if line.split()[1] == f"\"{self.modelname}\"":
                    line_reading_flag = True
            if not line_reading_flag:
                output.append(line)
            if line.startswith("))"):           # Eeschema template end
                line_reading_flag = False

        sym_file = open(self.kicad_nghdl_sym, 'w')
        for line in output:
            sym_file.write(line)
        sym_file.close()
        os.chdir(cwd)
        print("Leaving directory, ", self.lib_loc)

    def createSym(self):
        self.dist_port = 2.54         # Distance between two ports (mil)
        self.inc_size = 2.54          # Increment size of a block (mil)
        cwd = os.getcwd()
        os.chdir(self.lib_loc)
        print("Changing directory to ", self.lib_loc)

        # Removing ")" from "eSim_Nghdl.kicad_sym"
        file = open(self.kicad_nghdl_sym, "r")
        content_file = file.read()
        new_content_file = content_file[:-2]
        file.close()
        file = open(self.kicad_nghdl_sym, "w")
        file.write(new_content_file)
        file.close()

        # Appending new schematic block
        sym_file = open(self.kicad_nghdl_sym, "a")
        line1 = self.template["start_def"]
        line1 = line1.split()
        line1 = [w.replace('comp_name', self.modelname) for w in line1]
        self.template["start_def"] = ' '.join(line1)

        if os.stat(self.kicad_nghdl_sym).st_size == 0:
            sym_file.write(
                "(kicad_symbol_lib (version 20211014) " +
                "(generator kicad_symbol_editor)" + "\n\n"
            )  # Eeschema starter code

        sym_file.write(
            self.template["start_def"] + "\n" + self.template["U_field"] + "\n"
        )

        line3 = self.template["comp_name_field"]
        line3 = line3.split()
        line3 = [w.replace('comp_name', self.modelname) for w in line3]
        self.template["comp_name_field"] = ' '.join(line3)

        sym_file.write(self.template["comp_name_field"] + "\n")

        line4 = self.template["blank_field"]
        line4_1 = line4[0]
        line4_2 = line4[1]
        line4_1 = line4_1.split()
        line4_1 = [w.replace('blank_quotes', '""') for w in line4_1]
        line4_2 = line4_2.split()
        line4_2 = [w.replace('blank_quotes', '""') for w in line4_2]
        line4[0] = ' '.join(line4_1)
        line4[1] = ' '.join(line4_2)
        self.template["blank_qoutes"] = line4

        sym_file.write(line4[0] + "\n" + line4[1] + "\n")

        draw_pos = self.template["draw_pos"]
        draw_pos = draw_pos.split()

        draw_pos = \
            [w.replace('comp_name', f"{self.modelname}_0_1") for w in draw_pos]
        draw_pos[8] = str(
            float(draw_pos[8]) + float(self.findBlockSize() * self.inc_size)
        )
        draw_pos_rec = draw_pos[8]

        self.template["draw_pos"] = ' '.join(draw_pos)

        sym_file.write(
            self.template["draw_pos"] + "\n" + self.template["start_draw"] +
            " \"" + f"{self.modelname}_1_1\"" + "\n"
        )

        input_port = self.template["input_port"]
        input_port = input_port.split()
        output_port = self.template["output_port"]
        output_port = output_port.split()
        inputs = self.portInfo[0: self.input_length]
        outputs = self.portInfo[self.input_length:]

        inputs = self.char_sum(inputs)
        outputs = self.char_sum(outputs)

        total = inputs + outputs

        port_list = []

        # Set input & output port
        input_port[4] = draw_pos_rec
        output_port[4] = draw_pos_rec

        for i in range(total):
            if (i < inputs):
                input_port[9] = f"\"in{str(i + 1)}\""
                input_port[13] = f"\"{str(i + 1)}\""
                input_port[4] = \
                    str(float(input_port[4]) - float(self.dist_port))
                input_list = ' '.join(input_port)
                port_list.append(input_list)

            else:
                output_port[9] = f"\"out{str(i - inputs + 1)}\""
                output_port[13] = f"\"{str(i + 1)}\""
                output_port[4] = \
                    str(float(output_port[4]) - float(self.dist_port))
                output_list = ' '.join(output_port)
                port_list.append(output_list)

        for ports in port_list:
            sym_file.write(ports + "\n")
        sym_file.write(
            self.template["end_draw"] + "\n\n" + ")"
        )
        sym_file.close()
        os.chdir(cwd)

        print('Leaving directory, ', self.lib_loc)
        QtWidgets.QMessageBox.information(
            self.parent, "Symbol Added",
            '''Symbol details for this model is added to the \'''' +
            '''eSim_Nghdl.kicad_sym\' in the KiCad shared directory.''',
            QtWidgets.QMessageBox.Ok
        )

class PortInfo:
    '''
        The class contains port information
    '''
    def __init__(self, model):
        self.modelname = model.modelname
        self.model_loc = os.path.join(
            model.parser.get('NGHDL', 'DIGITAL_MODEL'), 'ghdl'
        )
        self.bit_list = []
        self.input_len = 0

    def getPortInfo(self):
        info_loc = os.path.join(self.model_loc, self.modelname + '/DUTghdl/')
        input_list = []
        output_list = []
        read_file = open(info_loc + 'connection_info.txt', 'r')
        data = read_file.readlines()
        read_file.close()

        for line in data:
            if re.match(r'^\s*$', line):
                pass
            else:
                in_items = re.findall(
                    "IN", line, re.MULTILINE | re.IGNORECASE
                )
                out_items = re.findall(
                    "OUT", line, re.MULTILINE | re.IGNORECASE
                )
            if in_items:
                input_list.append(line.split())
            if out_items:
                output_list.append(line.split())

        for in_list in input_list:
            self.bit_list.append(in_list[2])
        self.input_len = len(self.bit_list)
        for out_list in output_list:
            self.bit_list.append(out_list[2])


# http://www.cnblogs.com/alex3714/articles/5161349.html
# Python 使用logging模块记录日志涉及四个主要类，使用官方文档中的概括最为合适：
# logger提供了应用程序可以直接使用的接口；
# handler将(logger创建的)日志记录发送到合适的目的输出；
# filter提供了细度设备来决定输出哪条日志记录；
# formatter决定日志记录的最终输出格式。

# import logging
# logging.basicConfig(level=logging.DEBUG,
#                     format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
#                     datefmt='%a, %d %b %Y %H:%M:%S',
#                     filename='test.log',
#                     filemode='a')
# logging.debug('debug message')
# logging.info('info message')
# logging.warning('warning message') #默认级别
# logging.error('error message')
# logging.critical('critical message')

import logging
logger = logging.getLogger()# 创建一个handler，用于写入日志文件
fh = logging.FileHandler('test.log')# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh) #logger对象可以添加多个fh和ch对象
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)
logger.debug('logger debug message')
logger.info('logger info message')
logger.warning('logger warning message')
logger.error('logger error message')
logger.critical('logger critical message')


#%%
# from enum import auto
# import numpy as np
# import matplotlib as mpl
# import matplotlib.pyplot as plt
# import pandas as pd
# import seaborn as sns
# from sklearn.model_selection import train_test_split

from os import path
import numpy as np
import pandas as pd
# from pandas.plotting import scatter_matrix
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import StratifiedShuffleSplit

#%%
# DATA LOAD
data_filename = path.join('.', "KSI.csv")
df_ksi = pd.read_csv (data_filename)

#%%

## DATA EXPLORATION
# initial data exploration
print("First three records")
print(df_ksi.head(3))
#%%
print("Statistics")
print(df_ksi.describe())
#%%

print("Dimensions: ", df_ksi.shape)
print()
print("Types")
print(df_ksi.dtypes)
print("Col Names")
print(df_ksi.columns.values)
print("Non null count")
#%%
print(df_ksi.info())
#%%

# INITIAL VISUALIZATION
########################
# later we will sort out the graphs and use subplots
#

# See coords of the accidents
df_ksi.plot(kind="scatter", x="X", y="Y", alpha=0.1)
plt.title("Coords of Accidents")
plt.xlabel("Lattitude")
plt.ylabel("Longitude")
plt.show()

#%%

# Frequency histograms of important columns
df_ksi["DISTRICT"].hist(bins=30, figsize=(25,15))
plt.xlabel("DISTRICT")
plt.ylabel("freqeuncy")
plt.show()

#%%

df_ksi["ROAD_CLASS"].hist(bins=30, figsize=(25,15))
plt.xlabel("ROAD_CLASS")
plt.ylabel("freqeuncy")
plt.show()

#%%

df_ksi["ACCLOC"].hist(bins=30, figsize=(25,15))
plt.xlabel("ACCLOC")
plt.ylabel("freqeuncy")
plt.show()

#%%
df_ksi["TRAFFCTL"].hist(bins=30, figsize=(25,15))
plt.xlabel("TRAFFCTL")
plt.ylabel("freqeuncy")
plt.show()

#%%

df_ksi["VISIBILITY"].hist(bins=30, figsize=(25,15))
plt.xlabel("VISIBILITY")
plt.ylabel("freqeuncy")
plt.show()

#%%

df_ksi["LIGHT"].hist(bins=30, figsize=(25,15))
plt.xlabel("LIGHT")
plt.ylabel("freqeuncy")
plt.show()

#%%

df_ksi["RDSFCOND"].hist(bins=30, figsize=(25,15))
plt.xlabel("RDSFCOND")
plt.ylabel("freqeuncy")
plt.show()

df_ksi["ALCOHOL"].hist(bins=30, figsize=(25,15))
plt.xlabel("ALCOHOL")
plt.ylabel("freqeuncy")
plt.show()

df_ksi["ACCLASS"].hist(bins=30, figsize=(25,15))
plt.xlabel("ACCLASS")
plt.ylabel("freqeuncy")
plt.show()

df_ksi["IMPACTYPE"].hist(bins=40, figsize=(25,15))
plt.xlabel("IMPACTYPE")
plt.ylabel("freqeuncy")
plt.show()

#%%

ax= plt.subplot()
df_ksi["INVTYPE"].hist(bins=30, figsize=(25,15))
plt.xlabel("INVTYPE")
plt.ylabel("freqeuncy")
plt.setp(ax.get_xticklabels(), rotation=30, horizontalalignment='right')
plt.show()

#%%

df_ksi["INVAGE"].hist(bins=50, figsize=(25,15))
plt.xlabel("INVAGE")
plt.ylabel("freqeuncy")
plt.show()

ax= plt.subplot()
df_ksi["VEHTYPE"].hist(bins=40, figsize=(25,15))
plt.xlabel("VEHTYPE")
plt.ylabel("freqeuncy")
plt.setp(ax.get_xticklabels(), rotation=30, horizontalalignment='right')
plt.show()
#%%
# See pairplot
#sns.pairplot(df_ksi)
#plt.show()

#%%

# histogram of all the numerical columns
df_ksi.hist(bins=50, figsize=(20,15))
plt.show()

#%%

# Pie chart
fig1, ax1 = plt.subplots()
explode = [0,0.1,0]
df_ksi.groupby('ACCLASS').size().plot(kind='pie', explode=explode, autopct='%.2f%%', ax=ax1)
ax1.set_ylabel('Accident Class', size=16)
ax1.axis('equal')
plt.show()

#%%

fig1, ax1 = plt.subplots()
df_ksi.groupby('IMPACTYPE').size().plot(kind='pie', autopct='%.2f%%', ax=ax1)
ax1.set_ylabel('Impact Type', size=22)
ax1.axis('equal')
plt.show()

#%%

fig1, ax1 = plt.subplots()
df_ksi.groupby('DISTRICT').size().plot(kind='pie', autopct='%.2f%%', ax=ax1)
ax1.set_ylabel('District', size=16)
ax1.axis('equal')
plt.show()

#%%

fig1, ax1 = plt.subplots()
df_ksi.groupby('ACCLOC').size().plot(kind='pie', autopct='%.2f%%', ax=ax1)
ax1.set_ylabel('Collision Location', size=16)
ax1.axis('equal')
plt.show()

#%%

fig1, ax1 = plt.subplots()
df_ksi.groupby('TRAFFCTL').size().plot(kind='pie', autopct='%.2f%%', ax=ax1, figsize=(20,15))
ax1.set_ylabel('Traffic Control Type', size=16)
ax1.axis('equal')
plt.show()
## End of Data exploration

#%%
#@note
pd.crosstab(df_ksi["VEHTYPE"], df_ksi["ACCLASS"])
#%%
pd.crosstab(df_ksi["INJURY"], df_ksi["ACCLASS"])

#%%
## FEATURE SELECTION
#####################

df_ksi.columns.values

#%%

# initial feature selection (tentative)
columns_selected = ["HOUR", "TIME", "STREET1", "STREET2", "DISTRICT", "HOOD_ID", "TRAFFCTL", "VISIBILITY", "LIGHT", "RDSFCOND", "IMPACTYPE", "INVTYPE", "INVAGE", "VEHTYPE","LONGITUDE", "LATITUDE",  "ACCLASS"]

# output column
output_column = "ACCLASS"
#%%
dfksi_main = df_ksi[columns_selected]
dfksi_main.head(3)
#%%
dfksi_main.info()

#%%
# recheck the missing values
df_ksi.isnull().sum()
df_ksi.replace('<Null>', np.nan, inplace=True)

#%%
df_ksi.isnull().sum()

#%%

# some testing
df_ksi["IMPACTYPE"].unique()
df_ksi["IMPACTYPE"].value_counts()

#%%

# any rows with missing values left with features selected
df_ksi[df_ksi.isnull().any(axis=1)]

#%%

# since there are relatively low data with missing values, we can delete them for now
# later we want to check for the countinous data and fill them with mean value
# dfksi_main = dfksi_main.dropna()

#%%

# DATA SPLIT
############
from sklearn.model_selection import StratifiedShuffleSplit

RAND_SEED = 123
SPLIT_SIZE = 0.2  # 20% test

# using stratified shuffle split
splitter = StratifiedShuffleSplit(n_splits=1, test_size=SPLIT_SIZE, random_state=RAND_SEED)
#%%
# separating the feature and target/output
print("Data shape:", dfksi_main.shape)
dfksi_main_X = dfksi_main.drop(output_column, axis=1)  # X
dfksi_main_y = dfksi_main[output_column]  # y
print("features:", dfksi_main_X.shape)
print("output:", dfksi_main_y.shape)

#%%
# splitting the data in training/testing
for train_index, test_index in splitter.split(dfksi_main_X, dfksi_main_y):
    dfksi_train_X = dfksi_main_X.iloc[train_index]
    dfksi_train_y = dfksi_main_y.iloc[train_index]
    dfksi_test_X = dfksi_main_X.iloc[test_index]
    dfksi_test_y = dfksi_main_y.iloc[test_index]

# final split result
print("train X shape:", dfksi_train_X.shape)
print("train y shape:", dfksi_train_y.shape)
print("test X shape:", dfksi_test_X.shape)
print("test y shape:", dfksi_test_y.shape)

#%%

# BUILDING THE PIPELINE
#######################

from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer

#%%
dfksi_main.info()

#%%
# checking for categorical columns
# we have 2 numerical and 10 categorical data
dfksi_numerical_columns = ['HOUR', 'TIME', 'HOOD_ID', 'LATITUDE', 'LONGITUDE']
dfksi_categorical_columns = ['STREET1', 'STREET2', 'DISTRICT', 'TRAFFCTL', 'VISIBILITY', 'LIGHT',
       'RDSFCOND', 'IMPACTYPE', 'INVTYPE', 'INVAGE', 'VEHTYPE']

#%%

df_ksi_num=df_ksi.select_dtypes(include='number')
df_ksi_cat=dfksi_main.select_dtypes(include='object')
df_ksi_cat.columns
#%%
# create separate transformer pipeline for numerica and categorical values
numerical_pipeline = Pipeline(
    steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]
)
numerical_pipeline

#%%
catgorical_pipeline = Pipeline([
        ('imputer', SimpleImputer(missing_values=np.nan, strategy='most_frequent')),
        ('encoder', OneHotEncoder(handle_unknown="ignore")),
    ])
catgorical_pipeline

#%%
# creating column transformer with numeric and categorical transformers
pre_processor = ColumnTransformer(
    transformers=[
        ("numeric", numerical_pipeline, dfksi_numerical_columns),
        ("categoric",  catgorical_pipeline, dfksi_categorical_columns),
    ]
)
pre_processor

#%%

# dfksi_main = full_pipeline.fit_transform(housing)

#%%

## BUILDING THE MODELS
######################

## LOGISTIC REGRESSION
# Alamin Ahmed
# building and testing the logistic regression model
from sklearn.linear_model import LogisticRegression

# pipeline including the pre-processor
LR_pipeline = Pipeline(
    steps=[
        ("preprocessor", pre_processor),
        ("classifier", LogisticRegression(max_iter=500))
    ]
)

#%%
# fitting the model
LR_pipeline.fit(dfksi_train_X, dfksi_train_y)

# printing initial model score
print("model score with test data: %.3f" % LR_pipeline.score(dfksi_test_X, dfksi_test_y))

# %%
## fine tuning with the grid search cross validation
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
LR_grid_params = [
    {'classifier' : [LogisticRegression()],
    'classifier__penalty' : ['l1', 'l2'],
    'classifier__C' : np.logspace(-4, 4, 20),
    'classifier__solver' : ['liblinear']}
]
LR_grid_search = GridSearchCV(LR_pipeline, param_grid = LR_grid_params, cv = 5)
LR_grid_search.fit(dfksi_train_X, dfksi_train_y)

# checking the scores
print("Best Score:", LR_grid_search.best_score_)
print("Best Params:", LR_grid_search.best_params_)
LR_grid_search.best_estimator_

#%%
LR_pipeline_final = LR_grid_search.best_estimator_
LR_pipeline_final.fit(dfksi_train_X, dfksi_train_y)
print("Final Score model score: %.3f" % LR_pipeline_final.score(dfksi_test_X, dfksi_test_y))

#%%
# predicting
LR_predictions = LR_pipeline_final.predict(dfksi_test_X)

# printing confusion matrix
LR_confusion_matrix = confusion_matrix(dfksi_test_y, LR_predictions)
print('Confusion Matrix:\n', LR_confusion_matrix)
sns.heatmap(LR_confusion_matrix/np.sum(LR_confusion_matrix), annot=True,
            fmt='.2%', cmap='Blues')

#%%
## DECISION TREE
## Suraj Regmi
from sklearn.tree import DecisionTreeClassifier

# pipeline including the pre-processor
pipelineForHT = Pipeline(
    steps=[("preprocessor", pre_processor),
           ("classifier", DecisionTreeClassifier(random_state=39))]
)

#%%
## hyper parameter tunining
param_grid = {
    "classifier__criterion":["gini","entropy"],
    "classifier__max_depth":[2,4,8,10,12,14,18,20],
    }

# using Grid search for fine tuning
from sklearn.model_selection import GridSearchCV
grid = GridSearchCV(pipelineForHT, param_grid)
grid.fit(dfksi_train_X, dfksi_train_y)

# checking the scores
grid.best_score_
grid.best_params_
grid.cv_results_

#%%

pipeline3 = Pipeline(
    steps=[("preprocessor", pre_processor),
           ("classifier", DecisionTreeClassifier(criterion="entropy",max_depth=20,random_state=39))]
)
pipeline3.fit(dfksi_train_X, dfksi_train_y)
print("Decision Tree model score: %.3f" % pipeline3.score(dfksi_test_X, dfksi_test_y))

#%%

## SUPPORT VECTOR
##Adrian Posadas
from sklearn.svm import SVC
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
#%%

pipeline_svm = Pipeline(
    steps=[("preprocessor", pre_processor),
           ("classifier", SVC(random_state=53))]
)
pipeline_svm.fit(dfksi_train_X, dfksi_train_y)
#Define the grid search parameters
param_grid_svm = [
    {'classifier__kernel': ['linear', 'rbf', 'poly', 'sigmoid'],
    'classifier__C': [0.01,0.1, 1, 10, 100],
    'classifier__gamma': [0.01, 0.03, 0.1, 0.3, 1.0, 3.0],
    'classifier__degree':[2,3]},
  ]
param_grid_svm
#Create a grid search object
grid_search_svm = GridSearchCV(pipeline_svm,
                                  param_grid_svm, scoring='accuracy',
                                  refit = True,
                                  cv=3,
                                  verbose=1)
grid_search_svm
#Fit your training data to the gird search object
grid_search_svm.fit(dfksi_train_X, dfksi_train_y)
#Print out the best parameters
print(grid_search_svm.best_params_)
print(grid_search_svm.best_estimator_)
#Predict the test data using the fine-tuned model identified during grid search
best_model_svm = grid_search_svm.best_estimator_
#{'classifier__C': 1, 'classifier__degree': 2, 'classifier__gamma': 0.3, 'classifier__kernel': 'poly'}
print("SVM model score: %.3f" % pipeline_svm.score(dfksi_test_X, dfksi_test_y))
scores = cross_val_score(best_model_svm, dfksi_train_X, dfksi_train_y, scoring='accuracy', cv=5)
print ('Min:')
print (scores.min())
print ('Mean:')
print (scores.mean())
print ('Max:')
print (scores.max())
final_predictions = best_model_svm.predict(dfksi_test_X)
print('Classification Report:')
print(classification_report(dfksi_test_y, final_predictions))
#Confusion Matrix
print('Confusion Matrix:')
print(confusion_matrix(dfksi_test_y, final_predictions))

#Optimized SVM Model
pipeline_svm_optimized = Pipeline(
    steps=[("preprocessor", pre_processor),
           ("classifier", SVC(C=1, kernel='poly', degree=2, gamma=0.3, random_state=53))]
)
pipeline_svm_optimized.fit(dfksi_train_X, dfksi_train_y)
print("OptimizedSVM model score: %.3f" % pipeline_svm_optimized.score(dfksi_test_X, dfksi_test_y))

##END OF SVM

#%%

##Random Forest Regression
#Atakan Aytar

#%%
##### Dont run this cell since its only gridsearch no need to waste time
from sklearn.ensemble import RandomForestClassifier

#Cross val
param_grid = {
    "classifier__criterion":["gini","entropy"],
    "classifier__n_estimators": [100, 200],
    "classifier__max_depth" : [4,5,6,None],
    }
pipelineForRF = Pipeline(
    steps=[("preprocessor", pre_processor),
            ("classifier", RandomForestClassifier(random_state = 42))]
)
from sklearn.model_selection import GridSearchCV
grid2 = GridSearchCV(pipelineForRF, param_grid)
grid2.fit(dfksi_train_X, dfksi_train_y)
print(grid2.best_params_)
print(grid2.best_score_)
print(grid2.cv_results_)

#%%
from sklearn.ensemble import RandomForestClassifier
pipelineForRFoptimized = Pipeline(
    steps=[("preprocessor", pre_processor),
           ("classifier", RandomForestClassifier(n_estimators = 100, random_state = 42,max_depth=None,criterion="gini"))]
)
pipelineForRFoptimized.fit(dfksi_train_X, dfksi_train_y)
print("Random forest model score: %.3f" % pipelineForRFoptimized.score(dfksi_test_X, dfksi_test_y))

#%%
#End of random forest
#%%
#Neural Network Model - Jamaal Bernabe
#import MLP Classifier
print("Neural Network Model")
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.metrics import confusion_matrix

#Function to display metrics easily
def scoremodel(model):
    print("Training Accuracy: " , model.score(dfksi_train_X,dfksi_train_y))
    print("Test Accuracy: " , model.score(dfksi_test_X,dfksi_test_y))
    y_pred = model.predict(dfksi_test_X)
    print("Accuracy Matrix: " , confusion_matrix(dfksi_test_y, y_pred))
    print("Best Parameters: " ,model.best_params_)
    print("Best Score: " ,model.best_score_)
    # scores = cross_val_score(model, dfksi_train_X, dfksi_train_y, scoring='accuracy', cv=5)
    # print(k)
    # print ('Min:')
    # print (scores.min())
    # print ('Mean:')
    # print (scores.mean())
    # print ('Max:')
    # print (scores.max())

mlp = MLPClassifier(random_state=123)
pipe = Pipeline(steps=[('pre',pre_processor),('mlpc', mlp)])
pipe.fit(dfksi_train_X, dfksi_train_y)
#%%

#%%
# #GridSearch
param_grid = {
            'mlpc__max_iter': [500,600],
            'mlpc__hidden_layer_sizes': [(8,4,2),(4,2,1)],
            'mlpc__activation': ['tanh','relu'],
            'mlpc__solver': ['adam'],
            'mlpc__alpha': [0.0001,0.001, 0.01],
            'mlpc__learning_rate': ['constant','adaptive'],
}

rnd_grid = RandomizedSearchCV(estimator=pipe, param_distributions= param_grid, cv=3,n_jobs = -1,n_iter=10)
rnd_result = rnd_grid.fit(dfksi_train_X,dfksi_train_y)
scoremodel(rnd_result)
#%%
param_grid = {
            'mlpc__max_iter': [600],
            'mlpc__hidden_layer_sizes': [(8,4,2),(4,2,1)],
            'mlpc__activation': ['tanh'],
            'mlpc__solver': ['adam'],
            'mlpc__alpha': [0.001, 0.01],
            'mlpc__learning_rate': ['adaptive'],
}

#5 minutes
grid = GridSearchCV(estimator=pipe, param_grid=param_grid, cv=3, n_jobs = -1,scoring="accuracy",refit=True)
grid_result = grid.fit(dfksi_train_X,dfksi_train_y)
y_pred = grid_result.predict(dfksi_test_X)
scoremodel(grid_result)
#%%
#Graph
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn import metrics

ConfusionMatrixDisplay.from_estimator(grid_result,dfksi_train_X,dfksi_train_y)
ConfusionMatrixDisplay.from_predictions(y_pred,dfksi_test_y)

# fpr, tpr, thresholds = metrics.roc_curve(y_pred,dfksi_test_y, y_pred)
# roc_auc = metrics.auc(fpr, tpr)
# display = metrics.RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc,
#                                estimator_name='Neural Network')
# display.plot()
# plt.show()
# %%

#We pickle the model i choose the random forest for now we can change it later with the best performing one

import joblib
joblib.dump(pipelineForRFoptimized, "./models/randomforest.pkl")
joblib.dump(grid_result,"./models/NeuralNetwork.pkl")
joblib.dump(LR_pipeline_final,"LogReg.pkl")
joblib.dump(pipeline3,"./models/DecTree.pkl")
joblib.dump(pipeline_svm_optimized,"./models/SVM.pkl")
#%%
import json
inum = dfksi_test_y.index[dfksi_test_y == "Fatal"]
#print(inum)
num = dfksi_test_y.loc[dfksi_test_y == "Non-Fatal Injury"]

number = 3371
now_json = dfksi_test_X.iloc[number].to_json()
now_result = dfksi_test_y.iloc[number]
# out_file = open("mytest.json","w")
# json.dump(now_json,out_file)
# out_file.close()
print(now_json,now_result)
# %%

import joblib
from sklearn import metrics
forest = joblib.load("./models/randomforest.pkl")
nnetwork = joblib.load("./models/NeuralNetwork.pkl")
logreg = joblib.load("./models/LogReg.pkl")
dectree = joblib.load("./models/DecTree.pkl")
fsvm = joblib.load("./models/SVM.pkl")
# %%
import json
from sklearn import metrics

def report_gen(model,name):
    model_results = model.predict(dfksi_test_X)
    report = metrics.classification_report(dfksi_test_y,model_results,output_dict=True)
    #print(report)
    out_file = open("./reports/{}.json".format(name),"w")
    json.dump(report,out_file)
    out_file.close()
#model_results = nnetwork.predict(dfksi_test_X)
report_gen(nnetwork,"nnetwork_results")
report_gen(forest,"forest_results")
report_gen(logreg,"logreg_results")
report_gen(dectree,"dectree_results")
report_gen(fsvm,"fsvm_results")

# %%
import json
from sklearn import metrics
def report_gen(model,name):
    model_results = model.predict(dfksi_test_X)
    report = metrics.classification_report(dfksi_test_y,model_results)
    print(report)
    matrix = metrics.confusion_matrix(dfksi_test_y,model_results)
    print(matrix)

report_gen(nnetwork,"nnetwork_results")
report_gen(forest,"forest_results")
report_gen(logreg,"logreg_results")
report_gen(dectree,"dectree_results")
report_gen(fsvm,"fsvm_results")
# %%
def scoremodel(model):
    print("Training Accuracy: " , model.score(dfksi_train_X,dfksi_train_y))
    print("Test Accuracy: " , model.score(dfksi_test_X,dfksi_test_y))
    y_pred = model.predict(dfksi_test_X)
    print("Confusion Matrix: \n" , confusion_matrix(dfksi_test_y, y_pred))
    #print("Best Parameters: " ,model.best_params_)
    #print("Best Score: " ,model.best_score_)

models = [ (nnetwork, "NN"), (forest,"forest"), (logreg,"LR"), (dectree,"DecTree"),(fsvm,"SVM")]
for (x,y) in models:
    print(y)
    scoremodel(x)

#nnetwork.feature_importances_
#nnetwork.best_estimator_.named_steps["mlpc"].feature_importances_
# %%


def plot_section(profname,arg):

    import config
    import numpy as np
    import matplotlib.pyplot as plt
    from scipy.interpolate import griddata

    f_p  = open(config.sdir2+profname+'_p.txt','r')
    f_xy = open(config.tdir2+profname+'_traj.txt','r')

    if arg == 0:
        f_var  = open(config.sdir2+profname+'_t.txt','r')
    if arg == 1:
        f_var  = open(config.sdir2+profname+'_s.txt','r')

    p_arr, s_arr, t_arr, lat_arr, lon_arr, dist_arr = ([] for i in range(6))

    p_cnt = 0
    for p_line in f_p: # Loop over lines and extract variables of interest
        p_line = p_line.strip().split() # removes /n (strip) and converts str to list (split)
        s_line = f_var.readline().strip().split()
        t_line = f_xy.readline().strip().split()
        lat_arr.append(float(t_line[1]))
        lon_arr.append(float(t_line[2]))
        dist_arr.append(float(t_line[3]))

        k = 0

        for i in p_line:
            if float(i)<9000:
                p_arr.append(float(i))
                s_arr.append(float(s_line[k]))
                t_arr.append(dist_arr[-1])
                k = k + 1
        if p_cnt == 0:
            np_elem = np.size(p_arr)
        p_cnt = p_cnt + 1
    f_p.close()
    f_var.close()

    p_arr = -np.asarray(p_arr)
    s_arr = np.asarray(s_arr)
    t_arr = np.asarray(t_arr)

    # define grid.
    xi = np.linspace(0,max(t_arr),p_cnt)
    yi = np.linspace(min(p_arr),0,np_elem)

    # grid the data.
    zi = griddata((t_arr, p_arr), s_arr, (xi[None,:], yi[:,None]), method='linear')
    CS = plt.contour(xi,yi,zi,15,linewidths=0.5,colors='k')
    CS = plt.contourf(xi,yi,zi,15,cmap=plt.cm.jet)

    plt.colorbar() # draw colorbar
    plt.scatter(t_arr,p_arr,marker='o',c='b',s=1) # draw locations of observations
    plt.grid()
    plt.ylabel('Pressure (dbar)',fontsize=5)
    plt.xlabel('Distance (Km)',fontsize=5)

    if arg == 0:
        plt.title('Temperature section - Float: '+profname,fontsize=5)
    if arg == 1:
        plt.title('Salinity section - Float: '+profname,fontsize=5)

    return;


from collections import namedtuple
from pathlib import Path

# Dataset and results root directory
_DATASET_ROOT = Path(__file__).parent / 'data/source file'
_RESULTS_ROOT = Path(__file__).parent / 'data/labeled data'
_MAP_ROOT = Path(__file__).parent / 'data/mapped data'
_RESULTS_ROOT.mkdir(exist_ok=True)

Dataset = namedtuple('Dataset', ['name', 'root', 'versions', 'src', 'labeldata', 'mapdata'])

# Source codes and bug repositories
# DP-CNN use camel, jEdit, lucene, xalan, xerces, synapse, poi
camel = Dataset(
    'camel',
    _DATASET_ROOT / 'camel',
    [1.4, 1.6],
    [_DATASET_ROOT / 'camel/camel-1.4', _DATASET_ROOT / 'camel/camel-1.6'],
    [_RESULTS_ROOT / 'camel/camel-1.4.csv', _RESULTS_ROOT / 'camel/camel-1.6.csv'],
    [_MAP_ROOT / 'camel/camel-1.4.csv', _MAP_ROOT / 'camel/camel-1.6.csv']
)

jedit = Dataset(
    'jedit',
    _DATASET_ROOT / 'jedit',
    [4.0, 4.1],
    [_DATASET_ROOT / 'jedit/jedit-4.0', _DATASET_ROOT / 'jedit/jedit-4.1'],
    [_RESULTS_ROOT / 'jedit/jedit-4.0.csv', _RESULTS_ROOT / 'jedit/jedit-4.1.csv'],
    [_MAP_ROOT / 'jedit/jedit-4.0.csv', _MAP_ROOT / 'jedit/jedit-4.1.csv']
)

lucene = Dataset(
    'lucene',
    _DATASET_ROOT / 'lucene',
    [2.0, 2.2],
    [_DATASET_ROOT / 'lucene/lucene-2.0', _DATASET_ROOT / 'lucene/lucene-2.2'],
    [_RESULTS_ROOT / 'lucene/lucene-2.0.csv', _RESULTS_ROOT / 'lucene/lucene-2.2.csv'],
    [_MAP_ROOT / 'lucene/lucene-2.0.csv', _MAP_ROOT / 'lucene/lucene-2.2.csv']
)

xalan = Dataset(
    'xalan',
    _DATASET_ROOT / 'xalan',
    [2.5, 2.6],
    [_DATASET_ROOT / 'xalan/xalan-2.5', _DATASET_ROOT / 'xalan/xalan-2.6'],
    [_RESULTS_ROOT / 'xalan/xalan-2.5.csv', _RESULTS_ROOT / 'xalan/xalan-2.6.csv'],
    [_MAP_ROOT / 'xalan/xalan-2.5.csv', _MAP_ROOT / 'xalan/xalan-2.6.csv']
)

xerces = Dataset(
    'xerces',
    _DATASET_ROOT / 'xerces',
    [1.2, 1.3],
    [_DATASET_ROOT / 'xerces/xerces-1.2', _DATASET_ROOT / 'xerces/xerces-1.3'],
    [_RESULTS_ROOT / 'xerces/xerces-1.2.csv', _RESULTS_ROOT / 'xerces/xerces-1.3.csv'],
    [_MAP_ROOT / 'xerces/xerces-1.2.csv', _MAP_ROOT / 'xerces/xerces-1.3.csv']
)

synapse = Dataset(
    'synapse',
    _DATASET_ROOT / 'synapse',
    [1.1, 1.2],
    [_DATASET_ROOT / 'synapse/synapse-1.1', _DATASET_ROOT / 'synapse/synapse-1.2'],
    [_RESULTS_ROOT / 'synapse/synapse-1.1.csv', _RESULTS_ROOT / 'synapse/synapse-1.2.csv'],
    [_MAP_ROOT / 'synapse/synapse-1.1.csv', _MAP_ROOT / 'synapse/synapse-1.2.csv']
)

poi = Dataset(
    'poi',
    _DATASET_ROOT / 'poi',
    [2.5, 3.0],
    [_DATASET_ROOT / 'poi/poi-2.5', _DATASET_ROOT / 'poi/poi-3.0'],
    [_RESULTS_ROOT / 'poi/poi-2.5.csv', _RESULTS_ROOT / 'poi/poi-3.0.csv'],
    [_MAP_ROOT / 'poi/poi-2.5.csv', _MAP_ROOT / 'poi/poi-3.0.csv']
)

# Current dataset in use. (change this name to change the dataset)
DATASET = camel

#data after concat will be save to file CSV
_INPUT_ROOT = Path(__file__).parent / 'datainput'

Datainput = namedtuple('Datainput',['name', 'root', 'version', 'label', 'tradgene'])

savecamel = Datainput(
    'camel',
    _INPUT_ROOT / 'camel',
    [1.4, 1.6],
    [_INPUT_ROOT / 'camel/label/camel-1.4.csv', _INPUT_ROOT / 'camel/label/camel-1.6.csv'],
    [_INPUT_ROOT / 'camel/tradgene/camel-1.4.csv', _INPUT_ROOT / 'camel/tradgene/camel-1.6.csv']
)

savejedit = Datainput(
    'jedit',
    _INPUT_ROOT / 'jedit',
    [4.0, 4.1],
    [_INPUT_ROOT / 'jedit/label/jedit-4.0.csv', _INPUT_ROOT / 'jedit/label/jedit-4.1.csv'],
    [_INPUT_ROOT / 'jedit/tradgene/jedit-4.0.csv', _INPUT_ROOT / 'camel/tradgene/jedit-4.1.csv']
)

savelucene = Datainput(
    'lucene',
    _INPUT_ROOT / 'lucene',
    [2.0, 2.2],
    [_INPUT_ROOT / 'lucene/label/lucene-2.0.csv', _INPUT_ROOT / 'lucene/label/lucene-2.2.csv'],
    [_INPUT_ROOT / 'lucene/tradgene/lucene-2.0.csv', _INPUT_ROOT / 'lucene/tradgene/lucene-2.2.csv']
)

savexalan = Datainput(
    'xalan',
    _INPUT_ROOT / 'xalan',
    [2.5, 2.6],
    [_INPUT_ROOT / 'xalan/label/xalan-2.5.csv', _INPUT_ROOT / 'xalan/label/xalan-2.5.csv'],
    [_INPUT_ROOT / 'xalan/tradgene/xalan-2.5.csv', _INPUT_ROOT / 'xalan/tradgene/xalan-2.5.csv']
)

savecxerces = Datainput(
    'xerces',
    _INPUT_ROOT / 'xerces',
    [1.2, 1.3],
    [_INPUT_ROOT / 'xerces/label/xerces-1.2.csv', _INPUT_ROOT / 'xerces/label/xerces-1.3.csv'],
    [_INPUT_ROOT / 'xerces/tradgene/xerces-1.2.csv', _INPUT_ROOT / 'xerces/tradgene/xerces-1.3.csv']
)

savesynapse = Datainput(
    'synapse',
    _INPUT_ROOT / 'synapse',
    [1.1, 1.2],
    [_INPUT_ROOT / 'synapse/label/synapse-1.1.csv', _INPUT_ROOT / 'synapse/label/synapse-1.2.csv'],
    [_INPUT_ROOT / 'synapse/tradgene/synapse-1.1.csv', _INPUT_ROOT / 'synapse/tradgene/synapse-1.2.csv']
)

savepoi = Datainput(
    'poi',
    _INPUT_ROOT / 'poi',
    [2.5, 3.0],
    [_INPUT_ROOT / 'poi/label/poi-2.5.csv', _INPUT_ROOT / 'pot/label/poi-3.0.csv'],
    [_INPUT_ROOT / 'poi/tradgene/poi-2.5.csv', _INPUT_ROOT / 'poi/tradgene/poi-3.0.csv']
)

if __name__ == '__main__':

    print(savecamel.label)


import argparse
from torchvision.datasets import MNIST

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Preprocessing of MNIST dataset.")
    parser.add_argument(
        "-o", "--output",
        type=str,
        help="Path where to store preprocessed datasets.",
        default=None
    )
    # Syntactic sugar
    parser.add_argument(
        "-s", "--stage",
        type=str,
        help="Kind of dataset split to use.",
        default='train',
        choices=('train', 'test')
    )
    args = parser.parse_args()

    if args.stage == 'train':
        # Download and store training dataset
        MNIST(args.output, train=True, download=True)
    if args.stage == 'test':
        # Download and store test dataset
        MNIST(args.output, train=False, download=True)

    print(
        """
    ******************************
    * Called MNIST preprocessing *
    ******************************

    - Download dataset
    - Split the dataset in training and inference
    - Preprocess it
    - Store it to local filesystem

    """
    )


from collections import OrderedDict

from RMUtilsFramework.rmTimeUtils import rmTimestampToDateAsString
from RMDataFramework.rmForecastInfo import RMForecastInfo
from RMDataFramework.rmMixerData import RMMixerData
from rmDatabase import RMTable
from RMUtilsFramework.rmLogging import log

##-----------------------------------------------------------------------------------------------------
##
##
class RMMixerDataTable(RMTable):
    def initialize(self):
        self.database.execute("CREATE TABLE IF NOT EXISTS mixerData ("\
                                            "forecastID INTEGER NOT NULL, "\
                                            "forecastTimestamp INTEGER NOT NULL, "\
                                            "timestamp NUMERIC NOT NULL, "\
                                            "temperature DECIMAL DEFAULT NULL, "\
                                            "rh DECIMAL DEFAULT NULL, "\
                                            "wind DECIMAL DEFAULT NULL, "\
                                            "solarRad DECIMAL DEFAULT NULL, "\
                                            "skyCover DECIMAL DEFAULT NULL, "\
                                            "rain DECIMAL DEFAULT NULL, "\
                                            "et0 DECIMAL DEFAULT NULL, "\
                                            "pop DECIMAL DEFAULT NULL, "\
                                            "qpf DECIMAL DEFAULT NULL, "\
                                            "condition INTEGER DEFAULT NULL, "\
                                            "pressure DECIMAL DEFAULT NULL, "\
                                            "dewPoint DECIMAL DEFAULT NULL, "\
                                            "minTemp DECIMAL DEFAULT NULL, "\
                                            "maxTemp DECIMAL DEFAULT NULL, "\
                                            "minRH DECIMAL DEFAULT NULL, "\
                                            "maxRH DECIMAL DEFAULT NULL, "\
                                            "et0calc DECIMAL DEFAULT NULL, "\
                                            "et0final DECIMAL DEFAULT NULL, "\
                                            #"FOREIGN KEY(forecastID) REFERENCES forecast(ID), "\
                                            "PRIMARY KEY(forecastID, timestamp)"\
                                            ")")
        self.database.commit()

    def addRecords(self, forecastID, forecastTimestamp, values):
        if(self.database.isOpen()):
            valuesToInsert = [(forecastID,
                               forecastTimestamp,
                               value.timestamp,
                               value.temperature,
                               value.rh,
                               value.wind,
                               value.solarRad,
                               value.skyCover,
                               value.rain,
                               value.et0,
                               value.pop,
                               value.qpf,
                               value.condition,
                               value.pressure,
                               value.dewPoint,
                               value.minTemp,
                               value.maxTemp,
                               value.minRH,
                               value.maxRH,
                               value.et0calc,
                               value.et0final
                                ) for value in values]

            self.database.executeMany("INSERT INTO mixerData(forecastID, forecastTimestamp, timestamp, "\
                                      "temperature, rh, wind, solarRad, skyCover, rain, et0, pop, qpf, "\
                                      "condition, pressure, dewPoint, "\
                                      "minTemp, maxTemp, minRH, maxRH, et0calc, et0final) "\
                                      "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", valuesToInsert)
            self.database.commit()

    def deleteOlderDataByTimestampCollision(self, values):
        if(self.database.isOpen()):
            timestampsToDelete = [(value.timestamp,) for value in values]
            self.database.executeMany("DELETE FROM mixerData "\
                                  "WHERE timestamp=? ", timestampsToDelete)
            self.database.commit()

    def getRecordsByThreshold(self, minTimestamp = None, maxTimestamp = None, orderAsc = True, asDict = False):
        result = []
        if(self.database.isOpen()):
            cursor = None

            order = "ASC"
            if not orderAsc:
                order = "DESC"

            if(minTimestamp == None and maxTimestamp == None):
                cursor = self.database.execute("SELECT * FROM mixerData ORDER BY timestamp " + order)
            if(maxTimestamp == None):
                cursor = self.database.execute("SELECT * FROM mixerData WHERE timestamp=? ORDER BY timestamp " + order, (minTimestamp, ))
            else:
                cursor = self.database.execute("SELECT * FROM mixerData WHERE ?<=timestamp AND timestamp<? ORDER BY timestamp " + order,
                                    (minTimestamp, maxTimestamp, ))

            for row in cursor:
                mixerData = RMMixerData(row[2])
                mixerData.temperature = row[3]
                mixerData.rh = row[4]
                mixerData.wind = row[5]
                mixerData.solarRad = row[6]
                mixerData.skyCover = row[7]
                mixerData.rain = row[8]
                mixerData.et0 = row[9]
                mixerData.pop = row[10]
                mixerData.qpf = row[11]
                mixerData.condition = row[12]
                mixerData.pressure = row[13]
                mixerData.dewPoint = row[14]
                mixerData.minTemp = row[15]
                mixerData.maxTemp = row[16]
                mixerData.minRH = row[17]
                mixerData.maxRH = row[18]
                mixerData.et0calc = row[19]
                mixerData.et0final = row[20]

                result.append(mixerData)

        return result

    def getLastRecordsByThreshold(self, minTimestamp = None, maxTimestamp = None, orderAsc = True, asDict = False, noOfRecords = None):
        if asDict:
            result = OrderedDict()
        else:
            result = []

        if(self.database.isOpen()):
            cursor = None

            order = "ASC"
            if not orderAsc:
                order = "DESC"

            limit = ""
            if noOfRecords:
                limit = " LIMIT %d" % noOfRecords

            if minTimestamp is None and maxTimestamp is None:
                cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData GROUP BY timestamp ORDER BY timestamp " + order + limit)
            elif minTimestamp is None:
                cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData WHERE timestamp<=? GROUP BY timestamp ORDER BY timestamp " + order + limit, (maxTimestamp, ))
            elif maxTimestamp is None:
                cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData WHERE ?<=timestamp GROUP BY timestamp ORDER BY timestamp " + order + limit, (minTimestamp, ))
            else:
                cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData WHERE ?<=timestamp AND timestamp<=? GROUP BY timestamp ORDER BY timestamp " + order + limit,
                                    (minTimestamp, maxTimestamp, ))

            for row in cursor:
                mixerData = RMMixerData(row[3])
                mixerData.temperature = row[4]
                mixerData.rh = row[5]
                mixerData.wind = row[6]
                mixerData.solarRad = row[7]
                mixerData.skyCover = row[8]
                mixerData.rain = row[9]
                mixerData.et0 = row[10]
                mixerData.pop = row[11]
                mixerData.qpf = row[12]
                mixerData.condition = row[13]
                mixerData.pressure = row[14]
                mixerData.dewPoint = row[15]
                mixerData.minTemp = row[16]
                mixerData.maxTemp = row[17]
                mixerData.minRH = row[18]
                mixerData.maxRH = row[19]
                mixerData.et0calc = row[20]
                mixerData.et0final = row[21]

                if asDict:
                    result[row[3]] = mixerData
                else:
                    result.append(mixerData)

        return result

    def getRecordsByForecast(self, useInsertOrder = False):
        result = OrderedDict()
        if(self.database.isOpen()):
            cursor = None
            if useInsertOrder:
                cursor = self.database.execute("SELECT * FROM mixerData ORDER BY forecastID ASC, timestamp ASC")
            else:
                cursor = self.database.execute("SELECT * FROM mixerData ORDER BY forecastID DESC, timestamp ASC")

            for row in cursor:
                forecastID = row[0]
                forecastTimestamp = row[1]

                mixerData = RMMixerData(row[2])
                mixerData.temperature = row[3]
                mixerData.rh = row[4]
                mixerData.wind = row[5]
                mixerData.solarRad = row[6]
                mixerData.skyCover = row[7]
                mixerData.rain = row[8]
                mixerData.et0 = row[9]
                mixerData.pop = row[10]
                mixerData.qpf = row[11]
                mixerData.condition = row[12]
                mixerData.pressure = row[13]
                mixerData.dewPoint = row[14]
                mixerData.minTemp = row[15]
                mixerData.maxTemp = row[16]
                mixerData.minRH = row[17]
                mixerData.maxRH = row[18]
                mixerData.et0calc = row[19]
                mixerData.et0final = row[20]

                if forecastID in result:
                    result[forecastID]["values"].append(mixerData)
                else:
                    result[forecastID] = {"timestamp" : forecastTimestamp, "values": [mixerData, ]}
        return result

    def getRecordsForLastForecast(self):
        forecast = None
        values = None
        if(self.database.isOpen()):

            cursor = self.database.execute("SELECT * FROM mixerData WHERE forecastID=(SELECT MAX(forecastID) from mixerData) ORDER BY timestamp ASC")

            for row in cursor:
                if not forecast:
                    forecast = RMForecastInfo(row[0], row[1])
                    values = []

                mixerData = RMMixerData(row[2])
                mixerData.temperature = row[3]
                mixerData.rh = row[4]
                mixerData.wind = row[5]
                mixerData.solarRad = row[6]
                mixerData.skyCover = row[7]
                mixerData.rain = row[8]
                mixerData.et0 = row[9]
                mixerData.pop = row[10]
                mixerData.qpf = row[11]
                mixerData.condition = row[12]
                mixerData.pressure = row[13]
                mixerData.dewPoint = row[14]
                mixerData.minTemp = row[15]
                mixerData.maxTemp = row[16]
                mixerData.minRH = row[17]
                mixerData.maxRH = row[18]
                mixerData.et0calc = row[19]
                mixerData.et0final = row[20]

                values.append(mixerData)

        return forecast, values

    def getLastRecordForDayForSimulator(self, dayTimestamp):
        result = OrderedDict()

        if(self.database.isOpen()):
            minTimestamp = dayTimestamp
            maxTimestamp = dayTimestamp + 86400

            row = self.database.execute("SELECT * FROM mixerData WHERE ?<=timestamp AND timestamp<? GROUP BY timestamp ORDER BY forecastID DESC LIMIT 1",
                                (minTimestamp, maxTimestamp, )).fetchone()

            if row:
                mixerData = RMMixerData(row[2])
                mixerData.temperature = row[3]
                mixerData.rh = row[4]
                mixerData.wind = row[5]
                mixerData.solarRad = row[6]
                mixerData.skyCover = row[7]
                mixerData.rain = row[8]
                mixerData.et0 = row[9]
                mixerData.pop = row[10]
                mixerData.qpf = row[11]
                mixerData.condition = row[12]
                mixerData.pressure = row[13]
                mixerData.dewPoint = row[14]
                mixerData.minTemp = row[15]
                mixerData.maxTemp = row[16]
                mixerData.minRH = row[17]
                mixerData.maxRH = row[18]
                mixerData.et0calc = row[19]
                mixerData.et0final = row[20]

                mixerDataDict = {
                    row[2]: mixerData
                }

                result = {
                    "forecastID": row[0],
                    "forecastTimestamp": row[1],
                    "data": mixerDataDict
                }

        return result

    def getLastKnownConditionForDay(self, dayTimestamp):
        if(self.database.isOpen()):
            minTimestamp = dayTimestamp
            maxTimestamp = dayTimestamp + 86400

            row = self.database.execute("SELECT condition FROM mixerData WHERE ?<=timestamp AND timestamp<? AND condition IS NOT NULL ORDER BY forecastID DESC, forecastTimestamp DESC LIMIT 1",
                                (minTimestamp, maxTimestamp, )).fetchone()

            if row:
                return row[0]

        return None

    def deleteRecordsByDayThreshold(self, dayTimestamp, commit = True):
        if(self.database.isOpen()):
            self.database.execute("DELETE FROM mixerData WHERE timestamp<?", (dayTimestamp, ))
            if commit:
                self.database.commit()

    def deleteRecordsHistoryByDayThreshold(self, dayTimestamp, commit = True):
        if(self.database.isOpen()):

            # Test Query
            #
            # SELECT * FROM mixerData WHERE NOT EXISTS
            # (SELECT NULL FROM
            #   (SELECT forecastTimestamp fTs, timestamp dayTs FROM mixerData WHERE timestamp<1411096483
            #    EXCEPT SELECT DISTINCT MAX(forecastTimestamp) fTs, timestamp dayTs FROM mixerData WHERE timestamp<1411096483 GROUP BY timestamp
            #   ) toDelete
            #  WHERE mixerData.forecastTimestamp=toDelete.fTs and mixerData.timestamp=toDelete.dayTs
            # );

            query = "DELETE FROM mixerData WHERE EXISTS (SELECT NULL from "\
                                                "("\
                                                    "SELECT forecastTimestamp fTs, timestamp dayTs FROM mixerData WHERE forecastTimestamp<? "\
                                                        "EXCEPT "\
                                                    "SELECT DISTINCT MAX(forecastTimestamp) fTs, timestamp dayTs FROM mixerData WHERE forecastTimestamp<? GROUP BY timestamp "\
                                                ") toDelete "\
                                                "WHERE mixerData.forecastTimestamp=toDelete.fTs and mixerData.timestamp=toDelete.dayTs)"

            self.database.execute(query, (dayTimestamp, dayTimestamp, ))
            if commit:
                self.database.commit()

    def getLastRecordsForecast(self):
        if self.database.isOpen():
            record = self.database.execute("SELECT MAX(forecastID), MAX(forecastTimestamp) FROM mixerData").fetchone()
            return record[0], record[1]
        return None, None

    def clear(self, commit):
        if(self.database.isOpen()):
            self.database.execute("DELETE FROM mixerData")
            if commit:
                self.database.commit()

    def dump(self):
        if self.database.isOpen():
            cursor = self.database.execute("SELECT * FROM mixerData ORDER BY timestamp DESC, forecastID DESC")

            for row in cursor:
                log.debug("fID=%d, fTs=%s,  dTs=%s" % (row[0], rmTimestampToDateAsString(row[1]), rmTimestampToDateAsString(row[2])))


import json
import time
import nltk
from itertools import combinations

ISO_PATH = "../../data/parallel/isocodes.json"
isocodes = json.load(open(ISO_PATH))
isocode_values = set()
iso_keys = list(isocodes.keys())
for key in isocodes:
    isocode_values.add(isocodes[key])

INPUT_PATH = '../../data/parallel/master.json'
master_json = json.load(open(INPUT_PATH))
keys = master_json.keys()
print("Number of keys: ", len(keys))

def extract(arg, master_keys, iso_keys, isocodes):
    extracted_dict = {}
    for key in master_keys:
        story_versions = list(master_json[key].keys())
        story_versions_codes = []
        for s in story_versions:
            if s in iso_keys:
                story_versions_codes.append(isocodes[s])
            else:
                story_versions_codes.append("")

        multiway = True
        for q in arg:
            if q not in story_versions_codes:
                multiway = False

        if multiway:
            inside_json = {}
            for i in range(len(story_versions)):
                if story_versions_codes[i] in arg:
                    inside_json[story_versions_codes[i]] = master_json[key][story_versions[i]]

            extracted_dict[key] = inside_json

    return extracted_dict

# we need to run this file for all langpairs next.
# then, we run filter_unequal_stories.py on the outputs.
# for any n-tuple defined here.
with open('../../data/parallel/raw/stats.txt', 'w') as fp:
    pass

isocode_values_list = list(isocode_values)
iso_combinations = list(combinations(isocode_values_list, 2))
for queries in iso_combinations:
    extracted = extract(queries, keys, iso_keys, isocodes)
    keys_current = extracted.keys()
    num_stories = len(keys_current)
    if num_stories > 0:
        with open(f'../../data/parallel/raw/{queries[0]}-{queries[1]}.json', 'w') as f:
            json.dump(extracted, f)

        with open('../../data/parallel/raw/stats.txt', 'a') as fp:
            fp.write(str(queries))
            fp.write("\t"+ str(num_stories))
            fp.write("\n")


# /gpfs/home/yuk/anaconda3/envs/py36/bin/python3
# -*- coding: utf-8 -*-

# tempmax filepath /gpfs/home/yuk/dailymaxtemp/cru/gs-temp/gs_temp_max
# landcover filepath /gpfs/home/yuk/LAI/out_inter

import numpy
import os
from osgeo import gdal, gdalconst
from osgeo import ogr, osr
import math
import xlrd
import xlwt
import pandas
import random
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.interpolate import make_interp_spline

plt.rc('font', family='Times New Roman')
plt.rcParams["font.weight"] = "bold"
plt.rcParams["axes.labelweight"] = "bold"

file = r'/gs_temp_max/1982-1988_summer_temMAX_mean_CRUbias40.tif'
lai_data1 = gdal.Open(file)
im_width1 = int(lai_data1.RasterXSize)
im_height1 = int(lai_data1.RasterYSize)
band1 = lai_data1.GetRasterBand(1)  # bestmodel
image1 = band1.ReadAsArray(0, 0, im_width1, im_height1)
im_geotrans0 = lai_data1.GetGeoTransform()
im_proj0 = lai_data1.GetProjection()

file2 = r'/gs_temp_max/1988-1994_summer_temMAX_mean_CRUbias40.tif'
lai_data2 = gdal.Open(file2)
image2 = lai_data2.ReadAsArray(0, 0, im_width1, im_height1)

file3 = r'/gs_temp_max/1994-2000_summer_temMAX_mean_CRUbias40.tif'
lai_data3 = gdal.Open(file3)
image3 = lai_data3.ReadAsArray(0, 0, im_width1, im_height1)

file4 = r'/gs_temp_max/2000-2006_summer_temMAX_mean_CRUbias40.tif'
lai_data4 = gdal.Open(file4)
image4 = lai_data4.ReadAsArray(0, 0, im_width1, im_height1)

file5 = r'/gs_temp_max/2006-2012_summer_temMAX_mean_CRUbias40.tif'
lai_data5 = gdal.Open(file5)
image5 = lai_data5.ReadAsArray(0, 0, im_width1, im_height1)

file6 = r'/gs_temp_max/2012-2018_summer_temMAX_mean_CRUbias40.tif'
lai_data6 = gdal.Open(file6)
image6 = lai_data6.ReadAsArray(0, 0, im_width1, im_height1)

fileopt = r'/gs_temp_max/optima_cru_mean82-18_nonan.tif'
lai_data7 = gdal.Open(fileopt)
image7 = lai_data7.ReadAsArray(0, 0, im_width1, im_height1)

filecp = r'/gs_temp_max/timenode_maxtemp_GS_CRU.tif'
lai_data8 = gdal.Open(filecp)
image8 = lai_data8.ReadAsArray(0, 0, im_width1, im_height1)

file_veg = r'/out_inter/veg_mean0.2_0.5du.tif'
globe_data = gdal.Open(file_veg)
veg = globe_data.ReadAsArray(0, 0, im_width1, im_height1)

t1 = []
t2 = []
t3 = []
t4 = []
t5 = []
t6 = []
t7 = []
t8 = []

for ii in range(0, im_height1):
    for jj in range(0, im_width1):
        if veg[ii, jj] > 0:
            if (~numpy.isnan(image1[ii, jj])) and 0<image1[ii, jj]<40:
                t1.append(image1[ii,jj])
            else:
                continue
for ii in range(0, im_height1):
    for jj in range(0, im_width1):
        if veg[ii, jj] > 0:
            if (~numpy.isnan(image2[ii, jj])) and 0<image2[ii, jj]<40:
                t2.append(image2[ii, jj])
            else:
                continue

for ii in range(0, im_height1):
    for jj in range(0, im_width1):
        if veg[ii, jj] > 0:
            if (~numpy.isnan(image3[ii, jj])) and 0<image3[ii, jj]<40:
                t3.append(image3[ii, jj])
            else:
                continue
for ii in range(0, im_height1):
    for jj in range(0, im_width1):
        if veg[ii, jj] > 0:
            if (~numpy.isnan(image4[ii, jj])) and 0<image4[ii, jj]<40:
                t4.append(image4[ii, jj])
            else:
                continue

for ii in range(0, im_height1):
    for jj in range(0, im_width1):
        if veg[ii, jj] > 0:
            if (~numpy.isnan(image5[ii, jj])) and 0<image5[ii, jj]<40:
                t5.append(image5[ii, jj])
            else:
                continue

for ii in range(0, im_height1):
    for jj in range(0, im_width1):
        if veg[ii, jj] > 0:
            if (~numpy.isnan(image6[ii, jj])) and 0<image6[ii, jj]<40:
                t6.append(image6[ii, jj])
            else:
                continue
for ii in range(0, im_height1):
    for jj in range(0, im_width1):
        if veg[ii, jj] > 0:
            if (~numpy.isnan(image7[ii, jj])) and 0<image7[ii, jj]<40:
                t7.append(image7[ii, jj])
            else:
                continue

for ii in range(0, im_height1):
    for jj in range(0, im_width1):
        if veg[ii, jj] > 0:
            if (~numpy.isnan(image8[ii, jj])) and 0<image8[ii, jj]<40:
                t8.append(image8[ii, jj])
            else:
                continue
cmap = plt.get_cmap("Oranges")
f, ax = plt.subplots(figsize=(10, 5))
plt.subplots_adjust(left = 0.1, bottom=0.2, top = 0.8, right=0.8)

sns.distplot(t1, bins = [0,10,20,30,40], hist = False, kde = True, kde_kws = {'lw':1.5},norm_hist = False,rug = False, vertical = False,
             color = cmap(1/ 6), label = '82-88')
ax.axvline(x=numpy.nanmean(t1), color=cmap(1/ 6),linestyle='-', linewidth=2)

sns.distplot(t2, bins = [0,10,20,30,40], hist = False, kde = True, kde_kws = {'lw':1.5},norm_hist = False,rug = False, vertical = False,
             color = cmap(2/ 6), label = '88-94')
ax.axvline(x=numpy.nanmean(t2), color=cmap(2/ 6),linestyle='-', linewidth=2)

sns.distplot(t3, bins = [0,10,20,30,40], hist = False, kde = True, kde_kws = {'lw':1.5},norm_hist = False,rug = False, vertical = False,
             color = cmap(3/ 6), label = '94-00')
ax.axvline(x=numpy.nanmean(t3), color=cmap(3/ 6),linestyle='-', linewidth=2)

sns.distplot(t4, bins = [0,10,20,30,40], hist = False, kde = True, kde_kws = {'lw':1.5},norm_hist = False,rug = False, vertical = False,
             color = cmap(4/ 6), label = '00-06')
ax.axvline(x=numpy.nanmean(t4), color=cmap(4/ 6),linestyle='-', linewidth=2)

sns.distplot(t5, bins = [0,10,20,30,40], hist = False, kde = True, kde_kws = {'lw':1.5},norm_hist = False,rug = False, vertical = False,
             color = cmap(5/ 6), label = '06-12')
ax.axvline(x=numpy.nanmean(t5), color=cmap(5/ 6),linestyle='-', linewidth=2)

sns.distplot(t6, bins = [0,10,20,30,40], hist = False, kde = True, kde_kws = {'lw':1.5},norm_hist = False,rug = False, vertical = False,
             color = cmap(6/ 6), label = '12-18')
ax.axvline(x=numpy.nanmean(t6), color=cmap(6/ 6),linestyle='-', linewidth=2)

sns.distplot(t7, bins = [0,10,20,30,40], hist = False, kde = True, kde_kws = {'linestyle':'-.','lw':2.5},norm_hist = False,rug = False, vertical = False,
             color = 'deeppink', label = 'opt')
ax.axvline(x=numpy.nanmean(t7), color='deeppink',linestyle='-.', linewidth=2.5)

sns.distplot(t8, bins = [0,10,20,30,40], hist = False, kde = True, kde_kws = {'linestyle':'-.','lw':2.5},norm_hist = False,rug = False, vertical = False,
             color = 'darkturquoise', label = 'timenode')
ax.axvline(x=numpy.nanmean(t8), color='darkturquoise',linestyle='-.', linewidth=2.5)

ax.legend(frameon=False)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
ax.spines['left'].set_visible(False)
ax.spines['bottom'].set_visible(False)
# a = plt.axhline(y=0,ls="-",color=cm.binary(0.8), linewidth=3)
# a.set_zorder(0)
# sns.distplot(l2, bins = 20, hist = False, kde = True, kde_kws = {'color':'g', 'lw':2, 'linestyle':'--'},
#              norm_hist = False,rug = False, vertical = False,color = 'g', label = 'l1')
#ax.set_xlim([-0.1,0.1])
print(numpy.nanmean(t1))
print(numpy.nanmean(t2))
print(numpy.nanmean(t3))
print(numpy.nanmean(t4))
print(numpy.nanmean(t5))
print(numpy.nanmean(t6))
print(numpy.nanmean(t7))
print(numpy.nanmean(t8))

plt.show()
#


# import typing
from __future__ import annotations
from enum import Enum
from collections import namedtuple
from functools import cmp_to_key
from abc import ABC, abstractmethod

TaggedLine = namedtuple("TaggedLine", "line priorities")

class TreeWorker(ABC):
    @abstractmethod
    def accept(self, tree: Node):
        return tree

class TreeSorter(TreeWorker):
    def accept(self, tree: Node) -> Node:
        for child in tree.children:
            self.accept(child)
        tree.children.sort(key=cmp_to_key(lambda a, b: 0 if Node.is_irrelevant(a.tag) or Node.is_irrelevant(b.tag) else a.tag.value - b.tag.value))
        return tree

class TreeUnspacer(TreeWorker):
    # removing blank lines helps with the sorting
    def accept(self, tree: Node) -> Node:
        tree.children = [n for n in tree.children if n.tag != Tag.BLANK_LINE]
        for child in tree.children:
            self.accept(child)
        return tree

class TreeSpacer(TreeWorker):
    # doesn't work properly, but doesn't matter.
    # Running `gdformat script.gd` fixes spacing anyway.
    def accept(self, tree: Node) -> Node:
        tree.children = [c for c in tree.children if c.tag != Tag.BLANK_LINE]
        for c in tree.children:
            self.accept(c)

        for i in [i for i in range(len(tree.children)) if tree.children[i].children]:
            tree.children.insert(i, Node(Tag.BLANK_LINE, 1, "\n"))

        return tree

class TreePrinter(TreeWorker):
    def accept(self, tree: Node) -> str:
        s = tree.content
        for child in tree.children:
            s += self.accept(child)
        return s

class Node:
    @staticmethod
    def is_irrelevant(tag) -> bool:
        return tag in [Tag.IGNORE, Tag.BLANK_LINE]

    @staticmethod
    def node_comp(n1, n2) -> int:
        return 0 if Node.is_irrelevant(n1.tag) or Node.is_irrelevant(n2.tag) else n1.tag.value - n2.tag.value

    def __init__(self, tag, indent, content):
        self.tag = tag
        self.indent = indent
        self.parent = None
        self.children = []
        self.content = content

    def get_semantic_children(self):
        # only children that are not of type IGNORE
        return [child for child in self.children if not Node.is_irrelevant(child.tag)]

    def add_child(self, node: Node):
        self.children.append(node)
        node.parent = self

    def sort(self):
        for child in self.children:
            child.sort()
        self.children.sort(key=cmp_to_key(Node.node_comp))

    def __str__(self):
        return self.to_string("")

    def to_string(self, indent):
        s = "%s%s(\n" % (indent, self.tag)
        for c in self.children:
            s += c.to_string(indent + "  ")
            s += "\n"
        s += indent + ")"
        return s

class Tag(Enum):
    # https://docs.godotengine.org/en/stable/getting_started/scripting/gdscript/gdscript_styleguide.html#code-order
    ROOT = -1
    TOOL = 10
    CLASS_NAME = 20
    INHERITANCE = 30
    DOCSTRING = 40
    SIGNAL = 50
    ENUM = 60
    CONST = 70
    EXPORT_VAR = 80
    VAR = 90  # public and private
    ONREADY = 100
    FN_INIT = 110
    FN_READY = 120
    FN_BUILTIN = 130
    INNER_CLASS = 140
    STATIC_METHOD = 150
    METHOD = 160
    IGNORE = 1000
    BLANK_LINE = 2000

    def __lt__(self, other):
        return self.value < other.value

def indent_length(ident: str) -> int:
    return len(ident)  # hook in here for tabs vs spaces

# FIXME: make work for different spacing types
class TreeGenerator:
    def __init__(self):
        self._current_block = None

    def classify(self, line: str, line_number: int) -> Tag:
        tag = Tag.IGNORE
        if line.strip() == "":
            tag = Tag.BLANK_LINE
        elif line.startswith("tool"):
            tag = Tag.TOOL
        elif line.startswith("class_name"):
            tag = Tag.CLASS_NAME
        elif line.startswith("extends"):
            tag = Tag.INHERITANCE
        elif line.startswith("#"):
            children = self._current_block.get_semantic_children()
            # if either we have only encountered whitespace so far, or the last node was a docstring itself
            # (or another node that comes before that) then the line is a docstring too.
            # Else, it's just a regular comment → IGNORE
            tag = Tag.DOCSTRING if self._current_block.tag.value <= Tag.DOCSTRING.value and (len(children) == 0 or children[-1].tag.value <= Tag.DOCSTRING.value) else Tag.IGNORE
        elif line.startswith("const"):
            tag = Tag.CONST
        elif line.startswith("signal"):  # FIXME: check for looong signals
            tag = Tag.SIGNAL
        elif line.startswith("var"):
            tag = Tag.VAR
        elif line.startswith("export"):
            tag = Tag.EXPORT_VAR
        elif line.startswith("onready"):
            tag = Tag.ONREADY
        elif line.startswith("static func "):
            tag = Tag.STATIC_METHOD
        elif line.startswith("func _init"):
            tag = Tag.FN_INIT
        elif line.startswith("func _ready"):
            tag = Tag.FN_READY
        elif line.startswith("func _"):
            tag = Tag.FN_BUILTIN
        elif line.startswith("func "):
            tag = Tag.METHOD
        elif line.startswith("enum "):
            tag = Tag.ENUM
        elif line.startswith("class"):
            tag = Tag.INNER_CLASS

        return tag

    def treeify(self, lines):
        self._current_block = Node(Tag.ROOT, -1, "")

        for i in range(len(lines)):
            line = lines[i]
            stripped = line.lstrip()
            tag = self.classify(stripped, i + 1)
            indent = indent_length(line) - indent_length(stripped)
            if not line.endswith("\n"):
                # fix last line of file.
                # If that line is sorted to some other place, the
                # newline would be missing
                line += "\n"

            new_block = Node(tag, indent, line)
            if stripped:  # empty lines don't affect indent
                if indent == self._current_block.indent + 1:  # append to current parent block
                    self._current_block.add_child(new_block)
                elif indent == self._current_block.indent + 2:  # last block was actually a new parent block → fix up
                    self._current_block = self._current_block.children[-1]
                    self._current_block.add_child(new_block)
                    """
                    elif indent < self._current_block.indent:  # dedent → pop block(s)
                        while indent < self._current_block.indent:
                            self._current_block = self._current_block.parent
                        self._current_block.add_child(new_block)
                    """
                else:  # same level → new block
                    while indent < self._current_block.indent:
                        self._current_block = self._current_block.parent
                    assert indent == self._current_block.indent
                    self._current_block.parent.add_child(new_block)
                    self._current_block = new_block
            else:  # whitespace
                self._current_block.add_child(new_block)

        while self._current_block.parent is not None:
            self._current_block = self._current_block.parent

        return self._current_block

def main():
    tg = TreeGenerator()
    with open("./test/FogOfWar3D.gd", "r") as fh:
        lines = fh.readlines()
        tree = tg.treeify(lines)
        for worker in [TreeUnspacer(), TreeSorter(), TreePrinter()]:
            tree = worker.accept(tree)

        print(tree)

if __name__ == "__main__":
    main()


import streamlit as st
import pandas as pd
import base64

st.title('CSV Wrangler')

# File upload
uploaded_file = st.file_uploader("Choose a CSV file", type="csv")

# Read file and display in table
if uploaded_file is not None:
    df = pd.read_csv(uploaded_file)
    st.write(df)

    # Drop columns
    if st.checkbox('Drop columns'):
        columns = st.multiselect('Select columns to drop', options=df.columns)
        df = df.drop(columns=columns)
        st.write(df)

    # Filter rows
    if st.checkbox('Filter rows'):
        column = st.selectbox('Select a column', options=df.columns)
        value = st.text_input('Enter a value')
        filtered_df = df[df[column] == value]
        st.write(filtered_df)

    # Sort data
    if st.checkbox('Sort data'):
        column = st.selectbox('Select a column to sort by', options=df.columns)
        ascending = st.checkbox('Sort ascending')
        sorted_df = df.sort_values(by=column, ascending=ascending)
        st.write(sorted_df)

    # Download modified data
    if st.button('Download CSV'):
        csv = df.to_csv(index=False)
        b64 = base64.b64encode(csv.encode()).decode()
        href = f'Download CSV'
        st.markdown(href, unsafe_allow_html=True)


from xmlrpc.client import ServerProxy
import requests
from kafka import KafkaProducer
from bson.json_util import dumps, loads
from time import sleep

sleep(5)

producer = KafkaProducer(bootstrap_servers=['localhost:9092'],
                         value_serializer=lambda x:
                         dumps(x).encode('utf-8'))

ROCKETS_STATES_BASE_URL = "http://localhost:5000"
s = ServerProxy('http://localhost:9000')

if __name__ == "__main__":
    while True:
        command = input().split(' ')
        if (command[0] == "quit"):
            print("Bye")
            break
        if(command[0] == "startpoll"):
            response = s.getResponsesPoll(command[1],command[2])
            print(response)
        if(command[0] == "decide"):
            print("The rocket {} at {} : {}".format(command[3],command[2],command[1]))
        if(command[0] == "destroy"):
            print(command[1] + "\n" + command[2])
            # data = {'action' : "destroy",
            #     'siteName' : command[1],
            #         'rocketName' : command[2]}
            # producer.send('launcherTopic', value=data)
            responseDestruction = requests.put("{}/rocketsStates/destruction/{}/{}/{}".format(ROCKETS_STATES_BASE_URL, command[1], command[2], 1))

        if(command[0] == "launchSupplier"):
            response = s.launchSupplier(command[1],command[2])


from sklearn import datasets
from sklearn import svm

digit = datasets.load_digits()
# digit 데이터를 불러오는 것임.x
s=svm.SVC(gamma = 0.1, C = 10)
# s의 역할은 그저 빈 모델을 하나 생성하는 정도임.
s.fit(digit.data, digit.target)
# 이해를 못한 이유 : data는 피쳐(특징), target은 라벨(피쳐를 통해 나온값임.)
print(digit.data[0])
# 이걸로 나온 피쳐값 숫자 64개의 값으로 만들어진것이 아래것임.
print(digit.target[0])
# 이거는 위의 피쳐값 숫자 64개로 만들어진 숫자 0을 의미함. (이거 그려보면암)

new_d = [digit.data[0], digit.data[1], digit.data[2]]
# 그래서 각 피쳐값 3개를 넘겨줌.
result = s.predict(new_d)
# result에는 예측값을 던져주는 것임. 위의 3개의 피쳐값으로 나온 결과를 예측으로 던져줌

print("예측값 :", result)
print("참값 :", digit.target[0], digit.target[1], digit.target[2])
# 그래서 예측값(학습해서 나온거)이랑 원래 값을 비교해봄

result_2 = s.predict(digit.data)
# 이제 예측값으로 모든 피쳐값을 다 던져줌
correct = [i for i in range(len(result_2)) if result_2[i] == digit.target[i]]
accuracy = len(correct)/len(result_2)
print("정확도 : ", accuracy*100, "%")


"""Standard operation codes, for multiple numerical backends."""

from enum import IntEnum, unique

class RM(IntEnum):
    ROUND_NEAREST_EVEN = 0
    RNE = 0
    ROUND_NEAREST_AWAY = 1
    RNA = 1
    ROUND_UP = 2
    RTP = 2
    ROUND_DOWN = 3
    RTN = 3
    ROUND_TO_ZERO = 4
    RTZ = 4
    ROUND_AWAY_ZERO = 5
    RAZ = 5

class OF(IntEnum):
    INFINITY = 1
    INF = 1
    CLAMP = 2
    WRAP = 3

@unique
class OP(IntEnum):
    add = 0
    sub = 1
    mul = 2
    div = 3
    neg = 4
    sqrt = 5
    fma = 6
    copysign = 7
    fabs = 8
    fdim = 9
    fmax = 10
    fmin = 11
    fmod = 12
    remainder = 13
    ceil = 14
    floor = 15
    nearbyint = 16
    round = 17
    trunc = 18
    acos = 19
    acosh = 20
    asin = 21
    asinh = 22
    atan = 23
    atan2 = 24
    atanh = 25
    cos = 26
    cosh = 27
    sin = 28
    sinh = 29
    tan = 30
    tanh = 31
    exp = 32
    exp2 = 33
    expm1 = 34
    log = 35
    log10 = 36
    log1p = 37
    log2 = 38
    cbrt = 39
    hypot = 40
    pow = 41
    erf = 42
    erfc = 43
    lgamma = 44
    tgamma = 45


from daftlistings import Daft, RentType
import json

import results

def main():
    '''Search and provid option to contact each new ad'''
    with open('config/contact_details.json') as contact_file:
        contact_details = json.load(contact_file)

    new_listings = search_apts()
    num_of_listings = len(new_listings)

    for i, listing in enumerate(new_listings):
        print(f"\n\n******ADVERTISMENT {i+1} of {num_of_listings}******")
        print(f"{listing.description}\n€{listing.price}\n{listing.daft_link}")

        contact = input("Contact Ad? [y/n]: ")
        while contact.lower() not in {'y', 'n'}:
            contact = input('Options are only ["y", "n"]: ')

        if contact.lower() == 'y':
            contact_advertiser(listing, **contact_details)

def search_apts():
    '''
    Search Apts using the config file settings
    Returns
    -------
    list (listings)
    '''
    with open('config/search_config.json') as search_config_file:
        search_config = json.load(search_config_file)

    daft = Daft()
    daft.set_county(search_config['county'])
    daft.set_listing_type(RentType.APARTMENTS)
    daft.set_min_price(search_config['price']['min'])
    daft.set_max_price(search_config['price']['max'])

    listings = daft.search()

    # filter out seen results
    listings_seen = results.get_listings_seen_set()
    new_listings = [listing for listing in listings if listing.daft_link not in listings_seen]
    results.update_listings_seen(*listings)

    print(f"Found {len(new_listings)} new listings")

    return new_listings

def contact_advertiser(listing, *, name, phone, email, msg):
    '''Contact listing using contact_information'''
    contact = listing.contact_advertiser(
        name=name,
        contact_number=phone,
        email=email,
        message="\n".join(msg)
    )

    if contact:
        print("Advertiser contacted")

if __name__=="__main__":
    main()


import sys

import openai
openai.api_key = 'sk-xxx'

def openAIReq():
    # print('send request to openai_api')
    res = openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": sys.argv[1]},
        ]
    )
    # print('receive response from openai_api')
    print(res.choices[0].message.content)

openAIReq()