import sys
import os
import glob
import csv
import logging
from collections import OrderedDict
from util import *

def trade_dates(file):
    """Analyzes a trades csv file and returns a list of the corresponding dates
    that are covered in the file.
    """
    dates = []

    # Open the trades file and get each date
    with open(file, 'rb') as csvfile:
        reader = csv.DictReader(csvfile)

        for line in reader:
            date = int(float(line['date']))
            if not date in dates:
                dates.append(date)

    return dates

def order_quotes(dir):
    """Analyzes each of the dates in the quotes csv files and returns an ordered
    list of the files to execute from earliest to latest date and the corresponding
    dates and the stock tickers, with their line counts, that are covered in each file.
    """
    files = OrderedDict({})

    for file in glob.glob(os.path.join(os.path.normpath(dir), '*.csv')):
        with open(file, 'rb') as csvfile:
            files[file] = {}
            reader = csv.DictReader(csvfile)

            for line in reader:
                date = int(float(line['date']))
                if not date in files[file]:
                    files[file][date] = {line['SYMBOL']: reader.line_num}
                elif not line['SYMBOL'] in files[file][date]:
                    files[file][date][line['SYMBOL']] = reader.line_num

    return files

def write_taq(taq_output, file):
    """Writes the data in the the TAQ buffer to file, appending the TAQ buffer data
    to any existing entries.
    """
    # Write the headers of the file if the file does not exist
    if not os.path.isfile(file):
        with open(file, 'wb') as csvfile:
            writer = csv.DictWriter(csvfile, delimiter=',', fieldnames=taq_output[0].keys())
            writer.writeheader()

    with open(file, 'ab') as csvfile:
        writer = csv.DictWriter(csvfile, delimiter=',', fieldnames=taq_output[0].keys())
        writer.writerows(taq_output)

def calculate_NBBO(exchanges):
    """Calculates the national best bid and best offer of all exchanges
    and uses the corresponding bid size and offer size for each.

    Accepts a dictionary containing the exchanges and the corresponding
    BID, OFR, BIDSIZ, and OFRSIZ, for each.

    example: { 'N'  =>  { 'BID'     => 50.0,
                          'OFR'     => 48.00,
                          'BIDSIZ'  => 10,
                          'OFRSIZ'  => 5}
               ... }
    """
    # The current NBBO out of all exchanges
    NBBO = {'BID': 0, 'OFR': sys.maxint, 'BIDSIZ': 0, 'OFRSIZ': 0}

    # Get the best bid and offer
    for exchange in exchanges:
        # Ignore any bids or offers less than 1 USD (error in quotes file)
        if exchanges[exchange]['BID'] >= NBBO['BID'] and exchanges[exchange]['BID'] >= 1.0:
            NBBO['BID'] = exchanges[exchange]['BID']
            NBBO['BIDSIZ'] = exchanges[exchange]['BIDSIZ']
        if exchanges[exchange]['OFR'] <= NBBO['OFR'] and exchanges[exchange]['OFR'] >= 1.0:
            NBBO['OFR'] = exchanges[exchange]['OFR']
            NBBO['OFRSIZ'] = exchanges[exchange]['OFRSIZ']

    # If there is no valid OFR for the current NBBO set it to 0
    if NBBO['OFR'] == sys.maxint:
        NBBO['OFR'] = 0

    return NBBO

def add_taq_entry(taq_output, trade, NBBO, exchanges):
    """Creates a new TAQ entry and appends it to the TAQ output,
    the format of the new entry added to the TAQ output buffer is
    as follows:

    <All trade columns>, NBB, NBO, NBBSIZ, NBOSIZ, NQBB, NQBO, NQBBSIZ, NQBOSIZ

    If there is no available TAQ entry for NASDAQ then the columns in the TAQ output
    will contain 0 for each NSDQ entry.
    """
    taq_entry = OrderedDict({})

    # Add all trade data columns
    taq_entry['time']    = trade['time']
    taq_entry['symbol']  = trade['symbol']
    taq_entry['shares']  = trade['shares']
    taq_entry['buysell'] = trade['buysell']
    taq_entry['price']   = trade['price']
    taq_entry['type']    = trade['type']
    taq_entry['date']    = trade['date']

    # Add all NBBO columns
    if all(k in NBBO for k in ('BID', 'OFR', 'BIDSIZ', 'OFRSIZ')):
        taq_entry['NBB'] = NBBO['BID']
        taq_entry['NBO'] = NBBO['OFR']
        taq_entry['NBBSIZ'] = NBBO['BIDSIZ']
        taq_entry['NBOSIZ'] = NBBO['OFRSIZ']
    else:
        taq_entry['NBB'] = 0
        taq_entry['NBO'] = 0
        taq_entry['NBBSIZ'] = 0
        taq_entry['NBOSIZ'] = 0

    # Add the NASDAQ NBBO entries if they exist
    if 'T' in exchanges:
        taq_entry['NQBB'] = exchanges['T']['BID']
        taq_entry['NQBO'] = exchanges['T']['OFR']
        taq_entry['NQBBSIZ'] = exchanges['T']['BIDSIZ']
        taq_entry['NQBOSIZ'] = exchanges['T']['OFRSIZ']
    else:
        taq_entry['NQBB'] = 0
        taq_entry['NQBO'] = 0
        taq_entry['NQBBSIZ'] = 0
        taq_entry['NQBOSIZ'] = 0

    # Append the TAQ entry to the TAQ output buffer
    taq_output.append(taq_entry)

# The trades file, quotes directory, and resultant taq file
trades_file = ''
quotes_file = ''
taq_file = ''
log_file = ''

# Process command line arguments
if len(sys.argv) < 4:
    sys.stderr.write("Invalid arguments given\n")
    sys.stderr.write("Usage: align_quotes.py {trades file} {quotes file} {output file}\n")
    sys.stderr.write("Example: " + sys.argv[0] + "trades_08oct2012.csv quotes_08oct2012.csv taq_08oct2012.csv\n")
    sys.exit(1)

if os.path.isfile(sys.argv[1]) and os.path.isfile(sys.argv[2]):
    # Error if output file is directory
    if os.path.isdir(sys.argv[3]):
        sys.stderr.write("The output file cannot be a directory, please specify an output file!\n")
        sys.exit(1)

    # Error if trying to overwrite an existing output file
    if os.path.isfile(sys.argv[3]):
        sys.stderr.write("The output file already exists, please specify a different output file!\n")
        sys.exit(1)
    trades_file, quotes_file, taq_file = sys.argv[1:4]
    log_file = 'errors_' + os.path.basename(taq_file)
else:
    sys.stderr.write("Trades or quotes file does not exist, check arguments and try again!\n")
    sys.exit(1)

# Configure the logging
logging.basicConfig(format='', filename=log_file, level=logging.DEBUG)

# A buffer of the final taq data to be written to file, which is periodically flushed to disk
taq_output = list()

# For each entry in the trades file, iterate through the quotes files and produce a combined file
with open(trades_file, 'rb') as trades_csv:
    trades_reader = csv.DictReader(trades_csv)

    # Open the quotes file and read the first entry
    quotes_csv = open(quotes_file, 'rb')
    quotes_reader = csv.DictReader(quotes_csv)
    quote = quotes_reader.next()

    # Used to simplify processing duplicate trades file entries
    prev_symbol = None
    prev_trade_date = None
    prev_trade_time = None

    # The list of exchanges, each exchange has its own NBBO
    exchanges = {}
    NBBO = {}

    # Calculate current NBBO/NASDAQ BBO in the quotes file for each line in trades file
    for trade in trades_reader:
        symbol = trade['symbol']
        trade_date = float_to_int(trade['date'])
        trade_time = convert_mil(trade['time'])

        # Ignore the trade if it occurs when the markets are closed
        if not market_hours(trade_time):
            continue

        # If the current date and time is the same, use the previous results
        if prev_symbol == symbol and prev_trade_date == trade_date and prev_trade_time == trade_time:
            # Use the data for the current trade, duplicate the previous NBBO calculations
            add_taq_entry(taq_output, trade, NBBO, exchanges)
            continue
        else:
            prev_trade_date = trade_date
            prev_trade_time = trade_time

        # Reset the exchanges and NBBO for the new symbol
        if prev_symbol and (prev_symbol != symbol):
            exchanges.clear()
            NBBO.clear()

        # Set the previous symbol to the current
        prev_symbol = symbol

        # Iterate through the quotes file to the matching symbol entry
        while symbol > quote['SYMBOL']:
            try:
                quote = quotes_reader.next()
            except StopIteration:
                logging.warning(quotes_file + " : Ended before trades file :" + trades_file
                    + " at following trade: " + trade['symbol'] + ", " + trade_time.isoformat())
                break

        # If no matching entry in the quotes file add the most recent results, the NBBx results can
        # be all 0 in the case where there is no matching symbol in the quotes file
        if symbol < quote['SYMBOL']:
            add_taq_entry(taq_output, trade, NBBO, exchanges)
            continue

        # Get the NBBO for each exchange entry in the quotes file
        while (symbol == quote['SYMBOL']):
            try:
                quote_date = float_to_int(quote['date'])
                quote_time = convert_sec(quote['time'])

                # Ignore any quotes that occur when the markets are closed
                if not market_hours(quote_time):
                    quote = quotes_reader.next()
                    continue

                # Get the current NBBO for each exchange in the quotes file using the most recent entry
                if trade_date == quote_date and trade_time >= quote_time:
                    # Parse the latest NBBO bid, ofr, bidsiz, ofrsiz for each exchange
                    if not quote['EX'] in exchanges and quote['EX'] != 'D':
                        exchanges[quote['EX']] = {}

                    if quote['EX'] != 'D':
                        exchanges[quote['EX']]['BID'] = float(quote['BID'])
                        exchanges[quote['EX']]['OFR'] = float(quote['OFR'])
                        exchanges[quote['EX']]['BIDSIZ'] = float(quote['BIDSIZ'])
                        exchanges[quote['EX']]['OFRSIZ'] = float(quote['OFRSIZ'])

                # Error if NO matching entries in the quotes file has been found
                elif not exchanges and (trade_date < quote_date or trade_time < quote_time):
                    logging.warning(trades_file + " : No quotes entry found for the following trade: " + trade['symbol']
                        + ", " + trade_time.isoformat())
                    break
                # Break when the last matching quote entry for the time has been parsed
                elif exchanges and (symbol != quote['SYMBOL'] or trade_date < quote_date or trade_time < quote_time):
                    break

                # Read the next line of the quotes file
                quote = quotes_reader.next()

            # Error should not have reached the end of quotes file before trades file
            # TODO add support for the case where a single trades file spans across more than one quotes file
            except StopIteration:
                logging.warning(quotes_file + " : Ended before trades file :" + trades_file
                    + " at following trade: " + trade['symbol'] + ", " + trade_time.isoformat())
                break

        # Calculate the current NBBO out of all exchanges, add the results as a new entry in the taq output buffer
        NBBO = calculate_NBBO(exchanges)
        add_taq_entry(taq_output, trade, NBBO, exchanges)

        # If the TAQ buffer is > 250K lines, flush the buffer to disk
        if taq_output.__len__() >= 250000:
            write_taq(taq_output, taq_file)
            # Clear the buffer in memory
            del taq_output[:]

    # Close the open quotes files
    quotes_csv.close()

# Write any remaining content in the TAQ buffer to disk
if taq_output:
    write_taq(taq_output, taq_file)


# -*- coding: utf-8 -*-

import pytest
import requests
from jsonschema import validate

url_parts = ["", "/8094", "/search?query=Brewing",
             "/autocomplete?query=dog"]

@pytest.mark.parametrize("params", url_parts)
def test_status_code(base_url, params):
    """Проверка кода состояния HTTP"""
    target = base_url + params
    response = requests.get(target)
    assert response.status_code == 200

def test_api_json_schema(base_url):
    """Проверка схемы JSON для id = 9180"""
    resp = requests.get(base_url + "/9180")

    schema = {
        "type": "object",
        "properties": {
            "id": {"type": "number"},
            "obdb_id": {"type": "string"},
            "name": {"type": "string"},
            "brewery_type": {"type": "string"},
            "street": {"type": "string"},
            "address_2": {"type": "null"},
            "address_3": {"type": "null"},
            "city": {"type": "string"},
            "state": {"type": "string"},
            "country_province": {"type": "null"},
            "postal_code": {"type": "string"},
            "country": {"type": "string"},
            "longitude": {"type": "string"},
            "latitude": {"type": "string"},
            "phone": {"type": "null"},
            "website_url": {"type": "null"},
            "updated_at": {"type": "string"},
            "created_at": {"type": "string"}
        },
        "required": ["id", "obdb_id", "name", "brewery_type",
                     "city", "state", "country", "updated_at", "created_at"]
    }

    validate(instance=resp.json(), schema=schema)

# Типы пивоварен, передаваемые в качестве параметра в GET-запросе
types = ['micro', 'nano', 'regional', 'brewpub', 'large',
         'planting', 'bar', 'contract', 'proprietor', 'closed']

@pytest.mark.parametrize("brewery_type", types)
def test_sorting_by_type(base_url, brewery_type):
    """Проверка фильтра по значению параметра brewery_type"""
    target = base_url + f"?by_type={types}"
    resp = requests.get(target)
    for brewery in resp.json():
        assert brewery["brewery_type"] == types

def test_country_is_USA(base_url):
    """Проверяем, что страной является United States"""

    resp = requests.get(base_url)
    for brewery in resp.json():
        assert brewery["country"] == "United States"

@pytest.mark.skip
@pytest.mark.parametrize("amount",range(1, 50))
def test_parameter_per_page(base_url, amount):
    """Позитивный тест параметра per_page"""

    target = base_url + f"?per_page={amount}"
    resp = requests.get(target)
    assert len(resp.json()) == amount, "Количество пивоварен на странице "\
        "не соответствует переданному в GET-запросе параметру"

def test_border_conditions_per_page(base_url):
    """Проверка граничных условий параметра per_page"""

    res_1 = requests.get(base_url + "?per_page=0")
    res_2 = requests.get(base_url + "?per_page=51")

    assert res_1.json() == []
    assert len(res_2.json()) == 50, "Max per_page is not 50!"


from app import app
from flask import render_template, redirect, url_for, current_app
from app import db
from app.models import Car
from app.forms import AddForm, SearchForm
import sqlalchemy as sa
import sys

@app.route('/init')
def initialize_db():
    connection_uri = app.config['SQLALCHEMY_DATABASE_URI']
    engine = sa.create_engine(connection_uri)
    insp = sa.inspect(engine)

    # Create table only if it already doesn't exist
    if not insp.has_table("cars"):
        print('cars table does not exist.  Creating it.', file=sys.stdout)

        # Create the DB table
        db.create_all()

        # Iterate through the data file and add records to DB
        with current_app.open_resource('static/mpg.csv') as file:
            for row in file:
                # convert from bytes to string before manipulating
                toks = row.decode("utf-8").split(',')
                c = Car(name = toks[-1].strip(),
                        year = toks[-3],
                        origin = toks[-2],
                        mpg = toks[1])
                db.session.add(c)

        # Commit all the changes
        db.session.commit()

    return "Initialized DB"

@app.route('/wipe')
def wipe():
    db.drop_all()
    return "Wiped data."

@app.route('/')
def hello():
    return render_template('homepage.html')

@app.route('/view_all')
def view():
    all = db.session.query(Car).all()
    return render_template('view_cars.html', cars=all)

@app.route('/add', methods=['GET', 'POST'])
def add_record():
    form = AddForm()
    if form.validate_on_submit():
        # Extract values from form
        name_c = form.name.data
        year_c = form.year.data
        origin_c = form.origin.data
        mpg_c = form.mpg.data
        # Create a city record to store in the DB
        c = Car(name =name_c, year=year_c,origin=origin_c,mpg=mpg_c)

        # add record to table and commit changes
        db.session.add(c)
        db.session.commit()

        form.name.data = ''
        form.year.data = ''
        form.origin.data = ''
        form.mpg.data =''
        return redirect(url_for('add_record'))
    return render_template('add.html', form=form)

@app.route('/search_by_model', methods=['GET', 'POST'])
def search_by_name():
    form = SearchForm()
    if form.validate_on_submit():
        # Query DB table for matching name
        record = db.session.query(Car).filter(Car.name.contains(form.name.data))
        if record:
            return render_template('view_cars.html', cars=record)
        else:
            return render_template('not_found.html')
    return render_template('search.html', form=form)
@app.route('/sort_model')
def sort_model():
    all = db.session.query(Car).order_by(Car.name).all()
    return render_template('view_cars.html', cars=all)


#!/usr/bin/python3

from lxml import html
import requests
import hashlib
import base64
import re

session = "7uvlvl1ci3tho8pdl2pgrp7ag0"

# NOT GETTING THE RIGHT CHALLENGE

def process(data):
	print("="*35 + " Input " + "="*35)
	print(data)

#	can_decode = True
#	i = 0
#	while can_decode:
#		i += 1
#		try:
#			temp = base64.b64decode(data)
#			print(str(temp))
#			print("")
#			data = temp
#
#			if "\\" in str(data):
#				can_decode = False
#				print("Finished after " + str(i) + " decodes")
#		except:
#			can_decode = False
#			print("Finished after " + str(i) + " decodes")

	with open("file.txt", "w") as f:
		f.write(data)

	exit()

	ret = data

	print("-"*80)

	print(ret)
	return ret

def main():
	cookie = {'PHPSESSID': session}
	page = requests.get('https://ringzer0team.com/challenges/15', cookies=cookie)
	text = page.text

	if "Do not brute" in text:
		print("Login failed")
		exit()

	tree = html.fromstring(page.content)
	message = tree.xpath('/html/body/div[2]/div/div[2]/div/text()[2]').pop().strip()

	result = process(message)

	answerUrl = 'https://ringzer0team.com/challenges/15/' + result
	data = requests.get(answerUrl, cookies=cookie).content
	data=data.decode().split('<div class="alert alert-info">')

	print("")

	try:
		flag = re.findall(r"FLAG-\w+",data[1])
		print(flag)
	except:
		print("No flag")

main()


from typing import Dict, List
from app.utils import preprocessor_slot_description_to_value
import torch

from transformers import AutoTokenizer

from app.model.pretrained_models import get_pretrained_model, get_tokenizer

class FlanT5Sacc:
    def __init__(self, size: str, device) -> None:
        self.name = f"flan-t5-{size}"
        self.device = device
        model_path = f"google/flan-t5-{size}"
        self.model = get_pretrained_model(model_type="t5", model_path=model_path).eval().to(self.device)
        self.tokenizer: AutoTokenizer = get_tokenizer(tokenizer_type="t5", tokenizer_path=model_path)
        self.generation_config = {
            "max_new_tokens": 16,
        }

    def calculate(self, utterance: str, metadata: Dict) -> Dict[str, bool]:
        with torch.no_grad():
            parts = metadata["parts"]
            outputs = self.are_parts_correct_for_utterance(utterance, parts=parts)
            return {
                part["slot"]: output
                for part, output in zip(parts, outputs)
            }

    def are_parts_correct_for_utterance(self, utterance: str, parts: List[Dict]) -> List[bool]:
        utterance = self.preprocess_utterance(utterance)
        prompts = [
            self.get_prompt(utterance=utterance, part=part)
            for part in parts
        ]
        tokenized = self.tokenizer(
            prompts,
            max_length=256,
            padding="max_length",
            truncation=True,
            return_tensors="pt",
        ).to(self.device)
        input_ids = tokenized["input_ids"]
        attention_mask = tokenized["attention_mask"]
        output_tensor = self.model.generate(input_ids=input_ids, attention_mask=attention_mask, **self.generation_config)
        outputs = self.tokenizer.batch_decode(sequences=output_tensor, skip_special_tokens=True)
        assert len(outputs) == len(parts)
        outputs = [self.model_output_to_bool(output) for output in outputs]
        return outputs

    def get_prompt(self, utterance: str, part: Dict) -> str:
        template = self.preprocess_template(part["template"])
        if part["act"] in ["INFORM", "OFFER", "CONFIRM"]:
            return f'{utterance}\nCan we infer the following?\n{template}\n\n["yes", "no"]'
        elif part["act"] == "REQUEST":
            return f'{utterance}\nDo we ask the following question?\n"{template}"\n\["yes", "no"]'
        else:
            raise RuntimeError(f"Unknown act {part['act']}")

    def preprocess_utterance(self, utterance: str) -> str:
        return utterance

    def preprocess_template(self, template: str) -> str:
        return preprocessor_slot_description_to_value(template)

    def model_output_to_bool(self, output: str) -> bool:
        if output == "yes":
            return True
        elif output == "no":
            return False
        else:
            print(f"flan model returned '{output}' as a response")
            return False


import os

from fabric.api import cd, run, env, task, get
from fabric.contrib.files import exists

from ..repositories import get_repo_name
from fabric.context_managers import prefix

@task
def pip_download_cache(keep_dir=None):
    """Downloads pip packages into deployment pip dir.
    """
    if not exists(env.deployment_pip_dir):
        #         run('rm -rf {deployment_pip_dir}'.format(
        #             deployment_pip_dir=env.deployment_pip_dir))
        run('mkdir -p {deployment_pip_dir}'.format(
            deployment_pip_dir=env.deployment_pip_dir))
    with cd(env.project_repo_root):
        # can't use
        # run('pip download --python-version 3 --only-binary=:all: '
        # as not all packages have a wheel (arrow, etc)
        pip_download('pip')
        pip_download('setuptools')
        pip_download('ipython')
        pip_download('wheel')
        pip_download('gunicorn')
        run('pip3 download '
            '-d {deployment_pip_dir} -r {requirements}'.format(
                deployment_pip_dir=env.deployment_pip_dir,
                requirements=env.requirements_file), warn_only=True)

def pip_download(package_name):
    """pip downloads a package to the deployment_pip_dir.
    """
    run('pip3 download '
        '-d {deployment_pip_dir} {package_name}'.format(
            deployment_pip_dir=env.deployment_pip_dir,
            package_name=package_name), warn_only=True)

def pip_install_requirements_from_cache(venv_name=None):
    """pip installs required packages from pip_cache_dir into the venv.
    """
    package_names = get_required_package_names()
    for package_name in package_names:
        pip_install_from_cache(package_name=package_name, venv_name=venv_name)

def pip_install_from_cache(package_name=None, pip_cache_dir=None, venv_name=None):
    """pip install a package from pip_cache_dir into the venv.
    """
    pip_cache_dir = pip_cache_dir or env.deployment_pip_dir
    venv_name = venv_name or env.venv_name
    with cd(pip_cache_dir):
        # with prefix('workon {venv_name}'.format(venv_name=venv_name)):
        run('workon {venv_name} && pip3 install --no-cache-dir --no-index --find-links=. {package_name}'.format(
            venv_name=venv_name,
            package_name=package_name))

def get_required_package_names():
    package_names = []
    with cd(env.project_repo_root):
        data = run('cat {requirements}'.format(
            requirements=env.requirements_file))
        data = data.split('\n')
        for line in data:
            if 'botswana-harvard' in line or 'erikvw' in line:
                repo_url = line.split('@')[0].replace('git+', '')
                package_names.append(get_repo_name(repo_url))
    return package_names

def get_pip_list():
    local_path = os.path.expanduser(
        f'~/fabric/download/pip.{env.host}.{env.project_release}.txt')
    if os.path.exists(local_path):
        os.remove(local_path)
    remote_path = f'~/pip.freeze.{env.project_release}.txt'
    run(f'rm {remote_path}', warn_only=True)
    with prefix(f'source {env.venv_dir}/{env.project_appname}/bin/activate'):
        run(f'pip freeze > {remote_path}')
    get(remote_path=remote_path, local_path=local_path)


import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications import ResNet101
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, GlobalAveragePooling2D, Flatten, Dense
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau

x_train = np.load('../data/LPD_competition/npy/train_data.npy')
y_train = np.load('../data/LPD_competition/npy/label_data.npy')
print(x_train.shape)
print(y_train.shape)

y_train = to_categorical(y_train)

x_test = np.load('../data/LPD_competition/npy/test_data.npy')
print(x_test.shape)

x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=42)

train_generator = ImageDataGenerator(rescale=1./255, width_shift_range=0.1, height_shift_range=0.1).flow(x_train, y_train, batch_size=32)
val_generator = ImageDataGenerator(rescale=1./255, width_shift_range=0.1, height_shift_range=0.1).flow(x_val, y_val, batch_size=32)

test_generator = ImageDataGenerator(rescale=1./255).flow(x_test, shuffle=False)

resnet = ResNet101(weights='imagenet', include_top=False, input_shape=(128,128,3))
# resnet.summary()

resnet.trainable = False

input_tensor = Input(shape=(128,128,3))
layer = resnet(input_tensor)
layer = GlobalAveragePooling2D()(layer)
layer = Flatten()(layer)
layer = Dense(2048, activation='relu')(layer)
layer = Dense(1024, activation='relu')(layer)
output_tensor = Dense(1000, activation='softmax')(layer)

model = Model(inputs=input_tensor, outputs=output_tensor)

model.summary()

model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
path = './Lotte/resnet101_model.hdf5'
es = EarlyStopping(monitor='val_accuracy', patience=30)
cp = ModelCheckpoint(path, monitor='val_accuracy', save_best_only=True)
lr = ReduceLROnPlateau(monitor='val_accuracy', factor=0.8, patience=10)

model.fit(train_generator, epochs=2000, batch_size=32, validation_data=val_generator, callbacks=[es,cp,lr])

pred = model.predict(test_generator)
print(np.argmax(pred,1))

answer = pd.read_csv('./Lotte/sample.csv', header=0)
print(answer.shape)

answer.iloc[:,1] = np.argmax(pred,1)
print(answer)
answer.to_csv('./Lotte/submission.csv', index=False)

# 20.424


import torch
import torchvision
import torchvision.transforms as transforms

DATA_PATH = './data'

def get_transform():
    transform = transforms.Compose(
        [transforms.ToTensor(),
         transforms.Normalize((0.5,), (0.5,))])

    return transform

def get_dataset(transform, download=True):
    train_set = torchvision.datasets.MNIST(root=DATA_PATH, train=True, download=download, transform=transform)
    test_set  = torchvision.datasets.MNIST(root=DATA_PATH, train=False, download=download, transform=transform)
    return train_set, test_set

def get_dataloader(train_set, test_set, bs):
    trainloader = torch.utils.data.DataLoader(train_set, batch_size=bs, shuffle=True)
    testloader = torch.utils.data.DataLoader(test_set, batch_size=bs, shuffle=False)

    return trainloader, testloader


from django import forms
from django.contrib.contenttypes.models import ContentType
from django.db import transaction

from .fields import TaxiField, TaxiSingleField
from .models import TermTaxonomy, TermTaxonomyItem

class TaxiModelMixin(forms.ModelForm):
    """
    Mixin used on model forms where a TaxiField is set.
    The mixin is responsible to create and delete
    TermTaxonomyItems for a given instance based on form data.
    """

    taxi_fields = None

    def __init__(self, *args, **kwargs):
        self.taxi_fields = {}
        for field_name, field in self.__class__.base_fields.items():
            if type(field) in [TaxiField, TaxiSingleField]:
                self.taxi_fields[field_name] = field.taxonomy_slug

        # Set initial
        instance = kwargs.get("instance")
        if instance:
            kwargs["initial"] = kwargs.get("initial", {})
            for field in self.taxi_fields:
                kwargs["initial"].update(
                    {
                        field: list(
                            instance.terms.values_list("term_taxonomy__term", flat=True)
                        )
                    }
                )

        super().__init__(*args, **kwargs)

        # Set choices
        for field in self.taxi_fields:
            self.fields[field].choices = self.get_choices(field).values_list(
                "id", "term__name"
            )

    def get_choices(self, field):
        return TermTaxonomy.objects.filter(taxonomy__slug=self.taxi_fields[field])

    def get_existing_term_taxonomy_ids(self, field, instance):
        return list(
            instance.terms.filter(
                term_taxonomy__taxonomy__slug=self.taxi_fields[field]
            ).values_list("term_taxonomy", flat=True)
        )

    def _save_taxi(self):
        instance = self.instance
        content_type = ContentType.objects.get_for_model(instance)
        with transaction.atomic():
            for field, term_taxonomy in self.taxi_fields.items():
                existing = self.get_existing_term_taxonomy_ids(field, instance)
                choices = self.get_choices(field)
                data = self.cleaned_data[field]
                data = [int(data)] if type(data) == str else [int(i) for i in data]
                for choice in choices:
                    if choice.pk in data and choice.pk in existing:
                        # Do nothing, selected and already exists.
                        pass
                    elif choice.pk in data and choice.pk not in existing:
                        # Create the new choice that does not already exist.
                        TermTaxonomyItem(
                            content_type=content_type,
                            object_id=instance.pk,
                            term_taxonomy=choice,
                        ).save()
                    elif choice.pk not in data and choice.pk in existing:
                        # Remove the unselected choice that already exists.
                        TermTaxonomyItem.objects.filter(
                            term_taxonomy=choice,
                            content_type=content_type,
                            object_id=instance.pk,
                        ).delete()
                    elif choice.pk not in data and choice.pk not in existing:
                        # Final case, choice not selected and do not exist.
                        pass

    def _save_m2m_taxi(self):
        self._save_m2m()
        self._save_taxi()

    def save(self, commit=True):
        instance = super().save(commit=commit)
        if commit:
            self._save_taxi()
        else:
            self.save_m2m = self._save_m2m_taxi  # noqa

        return instance


#!/usr/bin/env python
# -*- coding: utf-8 -*-

def docclass_test():
    cl = fisherclassifier(getwords)
    cl.setdb('test1.db')
    sampletrain(cl)
    cl2 = naivebayes(getwords)
    cl2.setdb('test1.db')
    cl2.classify('quick money')

if __name__ == '__main__':
    import nose
    nose.main()


import serial
import time
import numpy as np
class DummyHead():

    def __init__(self, port):
        """Will work in deg in this class"""
        self.ino = serial.Serial('/dev/cu.usbmodem'+str(port), 115200, timeout=1)
        time.sleep(2)
        self.theta = 0

        #need big range here
        self.max_left = 30
        self.max_right = -30

        #Starting moving left
        self.left = True
        self.right = False

    def look_around(self):
        """Move the dummy hehad so it looks around randomly"""

        # go back and forth
        if self.left:
            self.theta += 1

            if self.theta >= self.max_left:
                self.left = False
                self.right = True

        elif self.right:
            self.theta -= 1

            if self.theta <= self.max_right: #past -90 deg
                self.left = True
                self.right = False

        # self.theta = 20
        self.move(self.theta)

    def get_rotation(self):
        return np.deg2rad(self.theta)

    def close(self):
        self.ino.close()

    def move(self, theta):
        """Moves dummy head to location"""

        q1 = theta
        q2 = 0

        msg = self.formatAngle(q1,q2)
        # print(msg)
        self.ino.write(msg)

    def formatAngle(self, q1,q2):
        q1, q2 = self.calibrate(q1, q2)
        data =  str(q1) + "," + str(q2) + '\n'
        data =  str(q1) + "," + str(q2) + '\n'
        return data.encode()

    def calibrate(self, q1_angle, q2_angle):
        #make this so
        q1_offset = 90 - 30
        q2_offset = 0

        # q1_angle *= -1
        return q1_angle + q1_offset, q2_angle + q2_offset
    def read(self):
        data = self.ino.readline()
        if data:
            data = data.decode('ascii')
            data = data.strip('\n')
            data = data.strip('\r')
            print(data)

# #Example Usage
# head = DummyHead(14141)
# angle = 0
# while True:
#     time.sleep(0.5)
#     head.move(90)
#     # head.move(90)
#     # angle += np.deg2rad(1)#hhead has 1 deg of res
#     head.read()
# head.close()


#Imports
from torch.utils.data import Dataset
import pandas as pd
import numpy as np
import librosa

import torch

class AudioLocationDataset(Dataset):
    def __init__(self, root="./../data_clip/", csv="./data_clip_label/label.csv", transform=None, use_subset=None, num_bin = 2):
        self.root = root
        self.root = "/Users/zachyamaoka/Dropbox/DE3/AudioExperinceDesign/de3_audio_data/data_clip/"
        self.csv = pd.read_csv(csv)
        if use_subset is not None:
            self.filenames = self.csv['Filename'].tolist()[:use_subset]
            self.labels = self.csv['Label'].tolist()[:use_subset]
        else:
            self.filenames = self.csv['Filename'].tolist()
            self.labels = self.csv['Label'].tolist()

        self.transform = transform
        self.num_bin = num_bin
        self.bins = get_bins(self.num_bin)

    def __len__(self):
        return len(self.filenames)

    def __getitem__(self, idx):
        #audio, label, rates = load_data_file(n=idx, audio_n_offset=0, label_rate=10, file_stem="real_rec_", data_label_path = "./data_real_label/", data_wav_path = "./../data_real_wav/")
        fname = self.filenames[idx]
        label = self.labels[idx]
        path = self.root + fname
        audio, sample_rate = librosa.core.load(path, sr=96000, mono=False)
        #print(audio[:, :5])
        #print(audio.shape)
        #print([label])
        #label = label[:, :2]
        #label = np.expand_dims(label, 1)

        #cut so they are all the same length
        # audio = audio[:, :192512]  #26146890 for synthetic

        # Take random 0.1 sample
        rate = 96000
        dur = 0.005
        chunk = int(rate*dur)

        max_rand_ind = 192512 - chunk - 1
        min_rand_ind = 0
        start = int(np.random.uniform(min_rand_ind,max_rand_ind))
        # print(start, start+chunk)
        audio = audio[:, start:(start+chunk)]

        #center data
        mean = np.mean(audio)
        audio -= mean

        #normalize

        max = np.max(np.abs(audio))
        audio /= max
        #label = label[:5995, :] #59291 for synthetic
        # if label<np.pi:
        #     label=[0]
        # else:
        #     label=[1]

        label = segment_data(label,self.bins) # convert from theta to bin

        label = [label]
        if self.transform:
            audio, label = self.transform((audio, label))

        return audio, label

def toPolar(xy):
    x = xy[0]
    y = xy[1]
    rho = np.sqrt(x**2 + y**2)
    phi = np.arctan2(y, x)
    return(rho, phi)

def toCartesian(rhophi):
    rho = rhophi[0]
    phi = rhophi[1]
    x = rho * np.cos(phi)
    y = rho * np.sin(phi)
    return(x, y)

def train_test_val_split(csv="./data_clip_label/label.csv", save_loc='./data_clip_label/'):
    csv = pd.read_csv(csv)
    filenames = csv['Filename'].tolist()
    labels = csv['Label'].tolist()
    all_i = np.arange(len(filenames))
    all_i = np.random.choice(all_i, len(all_i), replace=False)

    train=csv.sample(frac=0.8)
    testval=csv.drop(train.index)
    val=testval.sample(frac=0.5)
    test=testval.drop(val.index)

    print(type(train), '\n', train.head(), '\n len', len(train))
    print(type(val), '\n', val.head(), '\n len', len(val))
    print(type(test), '\n', test.head(), '\n len', len(test))

    train.to_csv(save_loc+"label_train.csv", index=False)
    test.to_csv(save_loc+"label_val.csv", index=False)
    test.to_csv(save_loc+"label_test.csv", index=False)

class ToTensor():
    def __init__(self):
        pass

    def __call__(self, sample):
        a, l = sample
        return torch.Tensor(a), torch.LongTensor(l)

def abs_radial_loss(h,y):
    global batch_size

    x = torch.abs(h.sub(y))
    x = torch.abs(x - np.pi)
    x = np.pi - x
    # print(x)
    # showind = np.random.randint(x.shape[0])
    # label = y.detach().numpy()[showind, 0]
    # pred = h.detach().numpy()[showind, 0]
    # x_ = x.detach().numpy()[showind, 0]
    # print("label: ", np.rad2deg(label), "pred: ", np.rad2deg(pred), "diff: ", np.rad2deg(x_))
    # time.sleep(3)
    # x = x * x #square difference
    x = torch.abs(x) # must be positive
    x = torch.sum(x)
    x = x/batch_size

    return x

def round_down(num, divisor):
    return num - (num%divisor)

def radial_loss(h, y):
    x = torch.abs(h.sub(y))
    x = torch.remainder(x, np.pi)
    x = torch.mean(x)
    return x

def get_bins(n):
    """ Splits heading into a discrete number of bins"""
    assert n % 2==0 # Onlys works for symetric bin numbers. Only symetric sub divsions are meaningful for the dummy headself.

    bin_size = (2 * np.pi)/n

    # Not tested, changes so bin go along middle.
    # how front how back......, saying front or back is easier.

    if n == 2:
        bin_offset = 0 # left right case
    else:
        bin_offset = bin_size/2

    start = 0
    theta = start + bin_offset
    bins = []
    for i in range(n):

        bins.append(theta)
        theta += bin_size

    return bins

def get_theta_quad(pred, n): #Rounds to center of quadrant
    """Coverts a pred label to theta value based on the number of quadrants"""

    if n == 2: #special case
        if pred == 0:
            return np.pi/2
        else:
            return 1.5 * np.pi

    else:
        step = 2 * np.pi/ n

        return (pred+1)* step
    #floors theta based on the number of bins

def segment_data(theta,bins):
    """segments polar data into respective bin and return ind, here 0 corresponds to the first bin. Assums data is between 0 and 2pi"""
    #bins of the from end
    n = len(bins)

    #find smallest starting value.

    for i in np.arange(n):
        j = i + 1
        if j > n - 1:
            return n - 1 #edge case
        if (theta >= bins[i]) and (theta <= bins[j]):
            return i #return the ind

# def abs_radial_loss(h,y):
#     global batch_size
#     # h = torch.remainder(h, np.pi) #
#     x = torch.abs(h.sub(y))
#     x = torch.abs(x - np.pi)
#     x = np.pi - x
#     x = x * x #square value
#
#     x = torch.sum(x) # average over batch
#     x = x / batch_size

    #return x


from typing import Callable, Dict

from starlette import status
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from starlette.responses import RedirectResponse, Response
from starlette.types import ASGIApp, Receive, Scope, Send

class LegacyRedirectMiddleware(BaseHTTPMiddleware):
    def __init__(
        self,
        app: ASGIApp,
        *,
        url_mapping: Dict[str, str],
    ) -> None:
        super().__init__(app)
        self.url_mapping = url_mapping

    async def dispatch(self, request: Request, call_next: Callable) -> Response:
        path = request.url.path

        if path not in self.url_mapping:
            response = await call_next(request)

            # User may have requested `/xyz/` when only `/xyz` is
            # in the URL mapping, resulting in a 404 false-positive.
            # Attempt mapping from `/xyz`.
            if response.status_code != 404 or not path.endswith("/"):
                return response

            path = path.rstrip("/")
            if path not in self.url_mapping:
                return response

        redirect_path = self.url_mapping[path]

        return RedirectResponse(
            request.url.replace(path=redirect_path),
            status_code=status.HTTP_301_MOVED_PERMANENTLY,
        )

class DomainRedirect:
    def __init__(
        self,
        domain: str,
        status_code: int = status.HTTP_301_MOVED_PERMANENTLY,
        root_path: str | None = None,
    ) -> None:
        self.domain = domain
        self.status_code = status_code
        self.root_path = root_path

    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
        assert scope["type"] == "http"

        request = Request(scope)
        url = request.url.replace(hostname=self.domain)
        if self.root_path:
            url = url.replace(path=f"{self.root_path}{request.url.path}")

        response = RedirectResponse(url, status_code=self.status_code)
        await response(scope, receive, send)


from pydoc import text
from flask import Flask, jsonify
import socket
import flask
import netifaces
import subprocess
from flask.globals import request
from uuid import getnode as get_mac
from threading import Thread
import os, sys, json, struct, socket, fcntl, time
import subprocess
from os import listdir
from os.path import islink, realpath, join
import re
import multiprocessing
import ipaddress

cli = sys.modules['flask.cli']
cli.show_server_banner = lambda *x: None

if "OBB_HOME" in os.environ:
    obb_home = os.environ["OBB_HOME"]
    reset_cmd = "make delete && make validate && sleep 5 && make deploy"
else:
    obb_home = "/opt/jci-obb"
    reset_cmd = "make delete && make validate && sleep 5 && make factory-deploy"

app = Flask(__name__)

@app.route('/')
def index():
    return '', 200

@app.route("/ping")
def ping():
    return jsonify({"status": "OK"}), 200

@app.post("/api/systemRestart")
def applyRestart():
    status = request.args.get('apply')
    if status == "true":
        def post_request_systemRestart():
            reboot_cmd = "sudo reboot"
            #reboot_cmd = "echo 'System Reboot'"
            time.sleep(10)
            subprocess.run([reboot_cmd],shell=True)
        threadRestart = Thread(target=post_request_systemRestart)
        threadRestart.start()
        return jsonify({'status': 202, 'message': "System Reboot Initiated Successfully"}), 202
    else:
        return jsonify({'status': 400, 'message': "Bad Request"}), 400

@app.post("/api/factoryReset")
def applyFactoryReset():
    status = request.args.get('apply')
    if status == "true":
        def post_request_factoryReset():
            factoryReset_cmd = "cd {} && {}".format(obb_home,reset_cmd)
            #factoryReset_cmd = "echo 'Factory Reset'"
            time.sleep(10)
            subprocess.run([factoryReset_cmd],shell=True)
        threadReset = Thread(target=post_request_factoryReset)
        threadReset.start()
        return jsonify({'status': 202, 'message': "Factory Reset Initiated Successfully"}), 202
    else:
        return jsonify({'status': 400, 'message': "Bad Request"}), 400

@app.route("/changeSettings",methods=['POST'])
def setStaticIp():
    request_data = request.get_json()

    valid_payload = isValidPayload(request_data)
    if not valid_payload :
        return jsonify({'status': 400, 'message': "Bad Request. Payload is not Valid!"}), 400

    dhcp_value=str(request_data.get("dhcpEnabled"))
    if dhcp_value=='False':
       dhc_false_add_addr="sudo netplan set ethernets.{interface}.addresses=[{ipaddr}/{mask}]".format(interface=request_data['name'],ipaddr=request_data['ipAddress'],mask=request_data['subnetMask'])
       subprocess.run(dhc_false_add_addr, capture_output=True, shell=True)
       autoDns = request_data["dns"]["auto"]
       dhc_false_dns_list=request_data["dns"]['nameservers']
       if autoDns == False:
          dhc_false_nameserver_add="sudo netplan set ethernets.{interface}.nameservers.addresses=[{nameserver}]".format(interface=request_data['name'],nameserver=",".join(dhc_false_dns_list))
          subprocess.run(dhc_false_nameserver_add, capture_output=True, shell=True)
       else:
          auto_dns_command="sudo netplan set ethernets.{interface}.nameservers.addresses={nameserver}".format(interface=request_data['name'],nameserver="null")
          subprocess.run(auto_dns_command, capture_output=True, shell=True)
       dhc_false_gateway_add="sudo netplan set ethernets.{interface}.gateway4={gatewayaddr}".format(interface=request_data['name'],gatewayaddr=request_data['defaultGateway'])
       subprocess.run(dhc_false_gateway_add, capture_output=True, shell=True)
       dhc_false_disable_cmd="sudo netplan set ethernets.{interface}.dhcp4={status}".format(interface=request_data['name'],status="no")
       subprocess.run(dhc_false_disable_cmd, capture_output=True, shell=True)
       pool = multiprocessing.Pool(processes=1)
       pool.apply_async(chconfig)
    elif dhcp_value=='True':
       dhc_true_command="sudo netplan set ethernets.{interface}.dhcp4={status}".format(interface=request_data['name'],status="yes")
       dhc_true_null_addr="sudo netplan set ethernets.{interface}.addresses={state}".format(interface=request_data['name'],state="null")
       dhc_tru_null_nameserver="sudo netplan set ethernets.{interface}.nameservers.addresses={state}".format(interface=request_data['name'],state="null")
       dhc_tru_null_gateway="sudo netplan set ethernets.{interface}.gateway4={state}".format(interface=request_data['name'],state="null")
       subprocess.run(dhc_true_command, capture_output=True, shell=True)
       subprocess.run(dhc_true_null_addr, capture_output=True, shell=True)
       subprocess.run(dhc_tru_null_nameserver, capture_output=True, shell=True)
       subprocess.run(dhc_tru_null_gateway, capture_output=True, shell=True)
       pool = multiprocessing.Pool(processes=1)
       pool.apply_async(chconfig)
    return "Response from API", 200

@app.route("/api/getstaticip", methods=['GET'])
def getStaticIp():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(("8.8.8.8", 80))
    staticip = s.getsockname()[0]
    s.close()
    return flask.jsonify(StaticIp=staticip)

@app.route("/api/getinterfaces", methods=['GET'])
def getinterfaces():
    if_res_main = []
    if_list = interdiscover()
    for if_name in if_list:
          if_data = {}
          if_data['name'] = if_name
          if_data['macAddress'] = getHwAddr(if_name)
          status = interfacestatus(if_name)
          if_data['status'] = status
          if status == 'Online':
            if_data['subnetMask'] = netifaces.ifaddresses(if_name)[netifaces.AF_INET][0]['netmask']
            if_data['ipAddress'] = netifaces.ifaddresses(if_name)[netifaces.AF_INET][0]['addr']
            gws=netifaces.gateways()
            if_data['defaultGateway'] = gws['default'][netifaces.AF_INET][0]
          else:
            if_data['subnetMask'] = "None"
            if_data['ipAddress'] = "None"
            if_data['defaultGateway'] = "None"
          if_data['dhcpEnabled'] = dhcpstatus(if_name)
          if_data['dns']= get_dns_settings(if_name)
          if_res_main.append(if_data)
    return flask.jsonify(if_res_main)

def interdiscover():
    all_interfaces = [i for i in listdir("/sys/class/net") if islink(join("/sys/class/net", i))]
    phy_interfaces = [i for i in all_interfaces if not realpath(join("/sys/class/net", i)).startswith(("/sys/devices/virtual", "/sys/devices/vif"))]
    return phy_interfaces

def dhcpstatus(intname):
    dhc_command="sudo netplan get ethernets.{}".format(intname)
    dhc_status = subprocess.run(dhc_command, capture_output=True, shell=True)
    x = dhc_status.stdout.decode()
    if "dhcp4: false" in x:
        return False
    else:
        return True

def interfacestatus(ifname):
    command = "sudo ethtool {}".format(ifname)
    ret = subprocess.run(command, capture_output=True, shell=True)
    x = ret.stdout.decode()
    if "Link detected: yes" in x:
        return "Online"
    else:
        return "Offline"

def getHwAddr(ifname):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    info = fcntl.ioctl(s.fileno(), 0x8927,  struct.pack('256s', bytes(ifname, 'utf-8')[:15]))
    return ':'.join('%02x' % b for b in info[18:24])

def get_dns_settings(ifname)-> dict:
    x=[]
    command = "sudo netplan get ethernets.{}.nameservers.addresses".format(ifname)
    ret = subprocess.run(command, capture_output=True, shell=True)
    x=ret.stdout.decode()
    z=re.findall( r'[0-9]+(?:\.[0-9]+){3}', x)
    return { 'nameservers': z, 'auto': len(z) == 0 }

def chconfig():
    apply_command="sudo netplan apply"
    subprocess.run(apply_command, capture_output=True, shell=True)
    subprocess.call(['sh', 'service-restart.sh'])
    return "shell executed successfully"

def isValidPayload(request_data):

    bool_array = ["True" , "False"]
    dhcpEnabled = str(request_data.get("dhcpEnabled"))
    autoDns = str(request_data["dns"]["auto"])
    interface_list = interdiscover()

    if(dhcpEnabled not in bool_array):
        return False

    if(autoDns not in bool_array):
        return False

    if(not str(request_data.get("name")) in interface_list):
        return False

    if(not str(request_data.get("subnetMask")).isdecimal):
        return False
    try:
        ipaddress.ip_address(str(request_data.get("ipAddress")))
        ipaddress.ip_address(str(request_data.get("defaultGateway")))
        dns_nameservers_list=request_data["dns"]["nameservers"]
        for nameserver in dns_nameservers_list:
            ipaddress.ip_address(nameserver)
    except ValueError:
         return False

    return True

if __name__ =='__main__':
    app.run(debug = False, host='0.0.0.0', port=8099)


import maya.cmds as cmds
from math import *

psphere = cmds.sphere(r=5)
pcube = cmds.polyCube()
pcone = cmds.polyCone()
closestToSurface = cmds.createNode("closestPointOnSurface")
cmds.connectAttr(closestToSurface+'.position', pcube[0]+'.translate')
cmds.connectAttr(pcone[0]+'.translate', closestToSurface+'.inPosition')
cmds.connectAttr(psphere[0]+'.worldSpace[0]',closestToSurface+'.inputSurface')


# App de Medição de Indice de Descarte

# Imports
import pickle
import numpy as np
import pandas as pd
import logging, io, os, sys
from sklearn.ensemble import GradientBoostingClassifier
from flask import Flask, render_template, flash, request, jsonify

# pip install flask_httpauth
from flask_httpauth import HTTPBasicAuth

# Cria a app
app = Flask(__name__)

# Inicializa a autenticação
auth = HTTPBasicAuth()

# Define o modelo como None
modelo_linhagem = None

# Variáveis de entrada
atributos = ['temperatura',
			 'amonia',
			 'idade',
			 'peso',
			 'sexo',
			 'fotoperiodo',
			 'ruido',
			 'luz',
			 'umidade',
			 'infeccoes',
			 'animais_por_gaiola',
			 'linhagem']

# Usuários de acesso
usuarios = {"ictb": "ictb", "lab": "lab"}

# Função para obter os usuários
@auth.get_password
def get_pw(username):
    if username in usuarios:
        return usuarios.get(username)
    return None

 # Função para mostrar imagem
def mostra_imagem(linhagem, nivel_descarte):

	# Verifica a linhagem
	if linhagem == 0:
		linhagem_str = 'black'
	else:
		linhagem_str = 'swiss'

	# Retorna a imagem correspondente
	return('/static/imagens/' + linhagem_str + '_' + str(nivel_descarte) + '.jpg')

# Função para carregar o modelo ao inicializar a app
@app.before_first_request
def startup():
	global modelo_linhagem

	# Carrega o modelo
	modelo_linhagem = pickle.load(open("static/modelo/modelo_linhagens.p",'rb'))

# Função para formatar mensagem de erro
@app.errorhandler(500)
def server_error(e):
    logging.exception('some error')
    return """
    And internal error <pre>{}</pre>
    """.format(e), 500

# Função para executar o processo de atribuição das variáveis
@app.route('/background_process', methods = ['POST', 'GET'])
def background_process():
	temperatura          = float(request.args.get('temperatura'))
	amonia               = float(request.args.get('amonia'))
	idade                = float(request.args.get('idade'))
	peso                 = float(request.args.get('peso'))
	sexo                 = float(request.args.get('sexo'))
	fotoperiodo          = float(request.args.get('fotoperiodo'))
	ruido                = float(request.args.get('ruido'))
	luz                  = float(request.args.get('luz'))
	umidade              = float(request.args.get('umidade'))
	infeccoes            = float(request.args.get('infeccoes'))
	animais_por_gaiola   = float(request.args.get('animais_por_gaiola'))
	linhagem             = int(request.args.get('linhagem'))

	# Cria o dataframe para os novos dados
	novos_dados = pd.DataFrame([[temperatura,
		                         amonia,
		                         idade,
		                         peso,
		                         sexo,
		                         fotoperiodo,
		                         ruido,
		                         luz,
		                         umidade,
		                         infeccoes,
		                         animais_por_gaiola,
		                         linhagem]],
		                         columns = atributos)

	# Faz as previsões
	previsoes = modelo_linhagem.predict_proba(novos_dados[atributos])

	# Obtém a melhor previsão (maior probabilidade)
	melhor_previsao = [3,6,9][np.argmax(previsoes[0])]

	# Retorna a cor do vinho e a imagem que corresponde à previsão
	return jsonify({'qualidade_prevista':melhor_previsao, 'image_name': mostra_imagem(linhagem, melhor_previsao)})

# Função para carregar a página principal e renderizar a imagem
@app.route("/", methods = ['POST', 'GET'])
@auth.login_required
def index():
	logging.warning("index!")
	return render_template('index.html', qualidade_prevista = 1, image_name = '/static/imagens/imagem.jpg')

# Executa a pp
if __name__ == '__main__':
    app.debug = True
    app.run()


import cv2
import sys
import  os

# 解析gpfpd消息
def parse_gpfpd(gpfpdfile):
    with open(gpfpdfile, 'r') as f:
        dic1 = []
        dic1.append(-1)
        dic2 = []
        dic2.append([0,0,0,0,-0.202,0,0,0,0,0,0,0,0,0,0,0])
        for line in f.readlines():
            line = line.strip('\n')
            # 去掉换行符\n
            b = line.split(',')
            # 将每一行以逗号为分隔符转换成列表
            dic1.append(b)
            #print(b)
        # 注意下列语句主要用于中间有空行的txt，
        for i in range(len(dic1)):
            if i%2 == 1:
                dic2.append(dic1[i])
    return dic2

# if __name__ == "__main__":
def rotate():
    start_index = 1
    end_index = 7366
    gpfpd = parse_gpfpd("TXT/imu1_0826.txt")
    print(gpfpd[1][1])
    print(gpfpd[2][1])

    img_dir = "PICTURE/photos1_0826_warped/"
    rotated_dir = "PICTURE/photos1_0826_rotated/"
    if not os.path.exists(rotated_dir):
        os.makedirs(rotated_dir)
    imgName_tail = ".jpg"
    # 设置图片从起始序号读取图片并进行旋转
    for i in range(start_index, end_index):
        print(i)
        img = cv2.imread(img_dir + str(i) + imgName_tail)
        rows, cols, chanel = img.shape
        # print(rows, cols)
        # rotate_angle = float(gpfpd[start_index][0])-float(gpfpd[2*i-1][0])
        # rotate_angle = float(357.71 - float(gpfpd[2 * i - 1][0]))
        rotate_angle = float(360 - float(gpfpd[i][3]))
        print(rotate_angle)
        M = cv2.getRotationMatrix2D(((cols - 1) / 2.0, (rows - 1) / 2.0), rotate_angle, 1)
        dst = cv2.warpAffine(img, M, (cols, rows))
        cv2.imwrite(rotated_dir + str(i) + imgName_tail, dst)
    return 1

    # 将图片以第一张图片为基准转正
    # for i in range(start_index, end_index):
    #     img = cv2.imread(rotated_dir + str(i) + imgName_tail)
    #     rows, cols, chanel = img.shape
    #     print(rows, cols)
    #     rotate_angle = 2
    #     print(rotate_angle)
    #     M = cv2.getRotationMatrix2D(((cols - 1) / 2.0, (rows - 1) / 2.0), rotate_angle, 1)
    #     dst = cv2.warpAffine(img, M, (cols, rows))
    #     cv2.imwrite("photo7_rotated_vertical/" + str(i) + imgName_tail, dst)


import urllib
from bs4 import BeautifulSoup
import re
import csv

url = "http://stats.footballpredictions.net/england/premier/1995-1996/results.html"
htmlfile = urllib.urlopen(url)
soup = BeautifulSoup(htmlfile)

#home team names
hometeam = soup.find_all("td", class_="hometeam")
ht = []
for element in hometeam:
	ht.append(element.a.get_text())

#away team names
awayteam = soup.find_all("td", class_="awayteam")
wt = []
for element in awayteam:
	wt.append(element.a.get_text())

#scores
scores = soup.find_all("td", class_="score")
sc = []
for element in scores:
	#sc.append(element.get_text())
	a = element.get_text()
	a = re.sub(r"\s+", "", a, flags=re.UNICODE)
	a = a.encode('ascii')
	sc.append(a.split('-'))

f = open("results.csv", "wt")

try:
	writer = csv.writer(f, delimiter=',')
	writer.writerow(("hometeam", "awayteam", "homegoal", "awaygoal"))
	for i in range(len(ht)):
		writer.writerow((ht[i], wt[i], sc[i][0], sc[i][1]))
finally:
	f.close()

f = open("results2.csv", "wt")

try:
	writer = csv.writer(f, delimiter=',')
	writer.writerow(("team", "opponent", "goal", "home"))
	for i in range(len(ht)):
		writer.writerow((ht[i], wt[i], sc[i][0], 1))
		writer.writerow((wt[i], ht[i], sc[i][1], 0))
finally:
	f.close()


import pandas as pd
from tqdm import tqdm
labels = ["Prevention", "Treatment", "Diagnosis", "Mechanism", "Case Report", "Transmission", "Forecasting", "General"]
df1 = pd.read_csv("org_proc.csv")
df2 = pd.read_csv("covid_dataset_shuffled.csv")
df2 = df2.sample(frac=1).reset_index(drop=True)
print(len(df2))
print(df1["processed_text"])
result = []
print(df2.columns)
cols = dict()
cols["processed_text"] = []
cols["ad_creative_body"] = []
for label in labels:
    cols[label] = []
for i, row_i in tqdm(df2.iterrows()):
    proc1 = row_i["processed_text"]
    result = df1.loc[df1["processed_text"] == proc1.strip()]
    if result.empty:
        continue
    cols["processed_text"].append(result["processed_text"].values[0].strip())
    cols["ad_creative_body"].append(result["ad_creative_body"].values[0])
    for label in labels:
        cols[label].append(row_i[label])
df3 = pd.DataFrame(cols)
print(df3)
print(len(df3))
df3.to_csv("sbp_dataset.csv")


from flask import jsonify, request
from flask_restful import Resource

from Model import db, VistorChainsTotal, VistorLevel, Vistor, States

from Model import VisitorChainTotalSchema, VistorLevelSchema, VisitorSchema, QuerySchema, StatesSchema, Names

from webargs import fields, validate
from webargs.flaskparser import use_args, use_kwargs, parser, abort

from sqlalchemy.orm import sessionmaker, load_only
from sqlalchemy import Column, Date, Integer, Text, create_engine, inspect, and_, or_
import datetime

currentDT = datetime.datetime.now()

visitor_chain_schema = VisitorChainTotalSchema
visitor_chain_schema = VisitorChainTotalSchema(many=True)

visitor_schema = VisitorSchema
visitor_schema = VisitorSchema(many=True)

visitor_level_schema = VistorLevelSchema
visitor_level_schema = VistorLevelSchema(many=True)

query_schema = QuerySchema
query_schema = QuerySchema(many=True)

schema_states = StatesSchema()
schema_states = StatesSchema(many=True)

class QueryResource(Resource):
	queryArgs = {
		"Level_1": fields.Str(),
		"Level_2": fields.Str(),
		"name": fields.Str(),
		"location": fields.Str(),
		"audience": fields.Str(),
		"frequency": fields.List(fields.Str()),
		"states": fields.List(fields.Str())
	}

	def _get(self, code):
		states = States.query.filter_by(abv=code).all()
		states = schema_states.dump(states).data
		return states

	def anayzedata(self, response):
		for x in response:
			y = x['ID']
			q = x['state']
			count = 0
			for z in response:
				if z['ID'] == y and z['state'] == q:
					count += 1
			x['count'] = count
		return response

	@use_args(queryArgs)
	def get(self, args):
		name = ""
		Level_1 = ""
		Level_2 = ""
		if 'name'in args:
			name = args['name']
		if 'Level_1'in args:
			level_1 = args['Level_1']
		if 'Level_2'in args:
			level_2 = args['Level_2']

		if 'location' in args and not args['location'] == "":
			res = Vistor.query.filter_by(LOCATION=args['location']).all()
			result = visitor_schema.dump(res).data
		elif 'Level_2' in args and not args['Level_2'] == "":
			res = Vistor.query.filter_by(Level_2=args['Level_2']).all()
			result = visitor_schema.dump(res).data

		elif 'Level_1' in args and not args['Level_1'] == "":
			res = Vistor.query.filter_by(Level_1=args['Level_1']).all()
			result = visitor_schema.dump(res).data
		else:
			res = Vistor.query.all()
			result = visitor_schema.dump(res).data

		audience = ""
		frequency = []
		states = []
		_response = []
		if 'audience' in args:
			audience = args['audience']
		if 'states' in args:
			states = args['states']
		if 'frequency' in args:
			frequency = args['frequency']
		if audience == 'states' and len(states) >= 1 and len(res) >= 1:
			for index, _rep in enumerate(result):
				if _rep['state'] in states:
					_response.append(_rep)
		else:
			_response = result
		if len(frequency) >= 1:
			frequencyMap = self.parseFrequency(frequency)
			if frequencyMap:
				if frequencyMap == [1, 0, 0]:
					_response = [i for i in _response if (i['Mild'] == 1)]
				elif frequencyMap == [0, 1, 0]:
					_response = [i for i in _response if (i['Moderate'] == 1)]
				elif frequencyMap == [0, 0, 1]:
					_response = [i for i in _response if (i['Frequent'] == 1)]
				elif frequencyMap == [1, 1, 0]:
					_response = [i for i in _response if (
						i['Mild'] == 1 and i['Moderate'] == 1)]
				elif frequencyMap == [1, 0, 1]:
					_response = [i for i in _response if (
						i['Mild'] == 1 and i['Frequent'] == 1)]
				elif frequencyMap == [0, 1, 1]:
					_response = [i for i in _response if (
						i['Moderate'] == 1 and i['Frequent'] == 1)]
		if len(_response) >= 1:
			for x, y in enumerate(_response):
				y['_map'] = self._get(y['state'])

		ananyzlized = self.anayzedata(_response)
		l_frengency = [i for i in range(0, len(frequency))]
		metadata = {
			'queryname': name,
			'queries': {
				'audience': audience,
				'Level_1': Level_1,
				'Level_2': Level_2,
				'frequecy': dict(zip(l_frengency, frequency)),
				'states': dict(zip([i for i in range(0, len(states))], states))
			},
			'time': str(datetime.datetime.now().timestamp())
		}
		return {"message": "Success", 'data': ananyzlized, 'meta': metadata}, 200

	@use_args(queryArgs)
	def get1(self, args):
		# _query = self.parseQuery(args)

		total = Vistor.query
		_level_1 = None
		_level_2 = None

		if not args["Level_1"] == "":
			_level_1 = args['Level_1']

		if not args['Level_2'] == "":
			_level_2 = args['Level_2']

		if type(_level_1) == str and type(_level_2) == str:
			total = total.filter_by(Level_1=_level_1, Level_2=_level_2)

		else:
			if type(_level_1) == str:
				total = total.filter_by(Level_1=_level_1)
			if type(_level_2) == str:
				total = total.filter_by(Level_2=_level_2)

		if 'location' in args and not args['location'] == "":
			_location = args['location']
			total = total.filter_by(LOCATION=_location)

		if 'frequency' in args:
			if(len(args['frequency'])) == 3:
				total = total.filter(
					or_(Vistor.Frequent == 1, Vistor.Mild == 1, Vistor.Moderate == 1))
			elif 'Frequent' in args['frequency']:
				total = total.filter_by(Frequent=1)
			elif 'Moderate' in args['frequency']:
				total = total.filter_by(Moderate=1)
			elif 'Mild' in args['frequency']:
				total = total.filter_by(Mild=1)
		if "states" in args and args['audience'] == "states":
			if len(args['states']) >= 1:
				pass
				# total = total.filter(and_(Vistor.state.in_([args['states']])))

	def parseFrequency(self, frequency):
		if len(frequency) == 3:
			return False
		elif 'Frequent' in frequency:
			_frequent = 1
		elif not 'Frequent' in frequency:
			_frequent = 0
		elif 'Moderate' in frequency:
			_moderate = 1
		elif not 'Moderate' in frequency:
			_moderate = 0
		elif 'Mild' in frequency:
			_mild = 1
		elif not 'Mild' in frequency:
			_mild = 0
		return [_mild, _moderate, _frequent]

	def object_as_dict(self, obj):
		return {c.key: getattr(obj, c.key)
				for c in inspect(obj).mapper.column_attrs}

	def _generateMaping(self, stateCode):
		_query = States.query.all()
		_query = StatesSchema.dump(_query).data
		return _query

class AudienceResource(Resource):
	queryArgs = {
		"Level_1": fields.Str(),
		"Level_2": fields.Str(),
		"name": fields.Str(),
		"location": fields.Str(),
		"audience": fields.Str(),
		"frequency": fields.List(fields.Str()),
		"states": fields.List(fields.Str())
	}
	def anayzedata(self, response, fmap):
		if fmap == [1, 1, 0] or [1,0,1] or [0,1,1]:
			for x in response:
				#y = x['ChainID']
				q = x['state']
				count = 0
				for z in response:
					if z['state'] == q:
						count += z['AudienceTotal']
				x['count'] = count
			return response
		else:
			return response

	def _get(self, code):
		states = States.query.filter_by(abv=code).all()
		states = schema_states.dump(states).data
		if len(states) >=1:
			return states[0]
		else:
			return {}
	def parseFrequency(self, frequency):
		_mild = 0
		_frequent = 0
		_moderate = 0
		if len(frequency) == 3:
			return False
		if 'Frequent' in frequency:
			_frequent = 1
		if 'Moderate' in frequency:
			_moderate = 1
		if 'Mild' in frequency:
			_mild = 1
		return [_mild, _moderate, _frequent]

	@use_args(queryArgs)
	def get(self, args):
		name = ""
		Level_1 = ""
		Level_2 = ""
		if 'name'in args:
			name = args['name']
		if 'Level_1'in args:
			level_1 = args['Level_1']
		if 'Level_2'in args:
			level_2 = args['Level_2']

		if 'location' in args and not args['location'] == "":
			res = VistorChainsTotal.query.filter_by(LOCATION=args['location']).all()
			result = visitor_chain_schema.dump(res).data
		elif 'Level_2' in args and not args['Level_2'] == "":
			res = VistorLevel.query.filter_by(Level_2=args['Level_2']).all()
			result = visitor_level_schema.dump(res).data

		elif 'Level_1' in args and not args['Level_1'] == "":
			res = VistorLevel.query.filter_by(Level_1=args['Level_1']).all()
			result = visitor_level_schema.dump(res).data
		else:
			res = VistorChainsTotal.query.all()
			result = visitor_chain_schema.dump(res).data
		audience = ""
		frequency = []
		states = []
		_response = []

		if 'audience' in args:
			audience = args['audience']
		if 'states' in args:
			states = args['states']
		if 'frequency' in args:
			frequency = args['frequency']
		if audience == 'states' and len(states) >= 1 and len(res) >= 1:
			for index, _rep in enumerate(result):
				if _rep['state'] in states:
					_response.append(_rep)
		else:
			_response = result
		frequencyMap=[]
		if len(frequency) >= 1:
			frequencyMap = self.parseFrequency(frequency)
			if frequencyMap:
				if frequencyMap == [1, 0, 0]:
					_response = [i for i in _response if (i['Mild'] == 1)]
				elif frequencyMap == [0, 1, 0]:
					_response = [i for i in _response if (i['Moderate'] == 1)]
				elif frequencyMap == [0, 0, 1]:
					_response = [i for i in _response if (i['Frequent'] == 1)]
				elif frequencyMap == [1, 1, 0]:
					_response = [i for i in _response if (
						i['Mild'] == 1 and i['Moderate'] == 1)]
				elif frequencyMap == [1, 0, 1]:
					_response = [i for i in _response if (
						i['Mild'] == 1 and i['Frequent'] == 1)]
				elif frequencyMap == [0, 1, 1]:
					_response = [i for i in _response if (
						i['Moderate'] == 1 and i['Frequent'] == 1)]

		if len(_response) >= 1:
			for x, y in enumerate(_response):
				y['_map'] = self._get(y['state'])

		ananyzlized = self.anayzedata(_response,frequencyMap)
		metadata = {
			'queryname': name,
			'queries': {
				'audience': audience,
				'Level_1': Level_1,
				'Level_2': Level_2,
				'frequecy': dict(zip([i for i in range(0,len(frequency))], frequency)),
				'states': dict(zip([i for i in range(0, len(states))], states))
			},
			'time': str(datetime.datetime.now().timestamp())
			}
		return {"message": "Success", 'args':args ,'data': ananyzlized, 'meta':metadata}, 200


from        optimize        import snopta, SNOPT_options
import      numpy           as np
from        scipy.spatial   import Delaunay
import      scipy.io        as io
import      os
import      inspect
from        dogs            import Utils
from        dogs            import interpolation
from        dogs            import constant_snopt_min

'''
 adaptiveK_snopt.py file contains functions used for DeltaDOGS(Lambda) algorithm.
 Using the package optimize (SNOPT) provided by Prof. Philip Gill and Dr. Elizabeth Wong, UCSD.

 This is a script of DeltaDOGS(Lambda) dealing with linear constraints problem which is solved using SNOPT.
 Notice that this scripy inplements the snopta function. (Beginner friendly)

 The adaptive-K continuous search function has the form:
 Sc(x) = (P(x) - y0) / K*e(x):

     Sc(x):     constant-K continuous search function;
     P(x):      Interpolation function:
                    For AlphaDOGS: regressionparameterization because the function evaluation contains noise;
                    For DeltaDOGS: interpolationparameterization;
     e(x):      The uncertainty function constructed based on Delaunay triangulation.

 Function contained:
     tringulation_search_bound:   Search for the minimizer of continuous search function over all the Delaunay simplices
                                  over the entire domain.
     Adoptive_K_search:           Search over a specific simplex.
     AdaptiveK_search_cost:       Calculate the value of continuous search function.

 LOG Dec. 4, 2018:   Function snopta still violates the constraints!
 LOG Dec. 4, 2018:   Put the actual constant into function bounds Flow and Fupp, do not include constant inside
                        function evaluation F(x).

 LOG Dec. 15, 2018:  The linear derivative A can not be all zero elements. Will cause error.

 LOG Dec. 16, 2018:  The 1D bounds of x should be defined by xlow and xupp,
                        do not include them in F and linear derivative A.

 LOG Dec. 18, 2018:  The 2D active subspace - DeltaDOGS with SNOPT shows error message:
                        SNOPTA EXIT  10 -- the problem appears to be infeasible
                        SNOPTA INFO  14 -- linear infeasibilities minimized
                     Fixed by introducing new bounds on x variable based on Delaunay simplex.

'''
##################################  adaptive K search SNOPT ###################################

def triangulation_search_bound_snopt(inter_par, xi, y0, ind_min, y_safe, L_safe, finest_mesh):
    # reddir is a vector
    inf = 1e+20
    n   = xi.shape[0]
    xE  = inter_par.xi
    # 0: Build up the Delaunay triangulation based on reduced subspace.
    if n == 1:
        sx = sorted(range(xi.shape[1]), key=lambda x: xi[:, x])
        tri = np.zeros((xi.shape[1] - 1, 2))
        tri[:, 0] = sx[:xi.shape[1] - 1]
        tri[:, 1] = sx[1:]
        tri = tri.astype(np.int32)
    else:
        options = 'Qt Qbb Qc' if n <= 3 else 'Qt Qbb Qc Qx'
        tri = Delaunay(xi.T, qhull_options=options).simplices
        keep = np.ones(len(tri), dtype=bool)
        for i, t in enumerate(tri):
            if abs(np.linalg.det(np.hstack((xi.T[t], np.ones([1, n + 1]).T)))) < 1E-15:
                keep[i] = False  # Point is coplanar, we don't want to keep it
        tri = tri[keep]
    # Sc contains the continuous search function value of the center of each Delaunay simplex

    # 1: Identify the minimizer of adaptive K continuous search function
    Sc = np.zeros([np.shape(tri)[0]])
    Scl = np.zeros([np.shape(tri)[0]])
    Sc_safe = np.zeros(tri.shape[0])

    Rmax, max_dis = constant_snopt_min.max_circumradius_delauany_simplex(xi, xE, tri)
    # Determine the parameters b and c for exterior uncertainty function.
    b, c, status  = constant_snopt_min.uncertainty_parameter_solver(Rmax, max_dis)

    for ii in range(np.shape(tri)[0]):
        R2, xc = Utils.circhyp(xi[:, tri[ii, :]], n)
        if R2 < inf:
            # initialize with body center of each simplex
            x     = np.dot(xi[:, tri[ii, :]], np.ones([n + 1, 1]) / (n + 1))
            exist = constant_snopt_min.unevaluated_vertices_identification(xi[:, tri[ii, :]], xE)[0]
            if exist == 0:  # The Delauany simplex considered is safe
                e = (R2 - np.linalg.norm(x - xc) ** 2)
            else:
                e = constant_snopt_min.discrete_min_uncertainty(x, xE, b, c)[0]

            Sc[ii]      = (interpolation.interpolate_val(x, inter_par) - y0) / e
            Sc_safe[ii] = (Sc[ii] if exist == 0 else inf)

            if np.sum(ind_min == tri[ii, :]):
                Scl[ii] = np.copy(Sc[ii])
            else:
                Scl[ii] = inf
        else:
            Scl[ii] = inf
            Sc[ii]  = inf

    if np.min(Sc) < 0:  # minimize p(x) subject to safe estimate
        Scp = np.zeros(tri.shape[0])
        for ii in range(tri.shape[0]):
            x = np.dot(xi[:, tri[ii, :]], np.ones([n + 1, 1]) / (n + 1))
            Scp[ii] = interpolation.interpolate_val(x, inter_par)
        ind = np.argmin(Scp)
        delta = 1.0e-5
        tol = 1.0e-4
        xmin, ymin, result = interpolation_min_safe_solver(xi[:, tri[ind, :]], inter_par, y0, y_safe, L_safe, delta, tol)

    else:
        # 2: Determine the minimizer of continuous search function at those 3 Delaunay simplices.
        # First index : Global one, the simplex that has minimum value of Sc at circumcenters.
        # Second index: Global one within the safe region.
        # Third index : Local one with the lowest interpolation value.
        index = np.array([np.argmin(Sc), np.argmin(Sc_safe), np.argmin(Scl)])
        xm    = np.zeros((n, 3))
        ym    = np.zeros(3)

        for i in range(3):
            temp_x, ym[i] = adaptivek_search_snopt_min(xi[:, tri[index[i], :]], inter_par, y0, y_safe, L_safe, b, c, finest_mesh)
            xm[:, i] = temp_x.T[0]
        ymin = np.min(ym)
        xmin = xm[:, np.argmin(ym)].reshape(-1, 1)
        if np.argmin(ym) == 0:
            result = 'sc global'
        elif np.argmin(ym) == 1:
            result = 'sc safe'
        else:
            result = 'sc local'
    val, idx, x_nn = Utils.mindis(xmin, xE)
    safe_estimate = y_safe[:, idx] - L_safe * val

    return xmin, ymin, result, safe_estimate
# =====================================  Continuous search function Minimization   ==================================

def adaptivek_search_snopt_min(simplex, inter_par, y0, y_safe, L_safe, b, c, finest_mesh):
    '''
    Find the minimizer of the search fucntion in a simplex using SNOPT package.
    The function F is composed as:  1st        - objective
                                    2nd to nth - simplex bounds
                                    n+1 th ..  - safe constraints
    :param simplex  :     Delauany simplex of interest, n by n+1 matrix.
    :param inter_par:     Interpolation info.
    :param y0       :     Target value of truth function
    :param y_safe   :     Safe function evaluation.
    :param L_safe   :     Lipschitz constant of safety functions.
    :param b        :     The parameters for exterior uncertainty function. It is determined once Delaunay-tri is fixed.
    :param c        :     The parameters for exterior uncertainty function. It is determined once Delaunay-tri is fixed.
    :return:              The minimizer of adaptive K continuous search function within the given Delauany simplex.
    '''
    inf = 1.0e+20
    xE  = inter_par.xi
    n   = xE.shape[0]
    M   = y_safe.shape[0]

    # -------  ADD THE FOLLOWING LINE WHEN DEBUGGING --------
    # simplex = xi[:, tri[ind, :]]
    # -------  ADD THE FOLLOWING LINE WHEN DEBUGGING --------

    # Determine if the boundary corner exists in simplex, if boundary corner detected:
    # e(x) = (|| x - x' || + c )^b - c^b,  x' in S^k
    # else, e(x) is the regular uncertainty function.
    exist, eval_indicators = constant_snopt_min.unevaluated_vertices_identification(simplex, xE)

    R2, xc = Utils.circhyp(simplex, n)
    x = np.dot(simplex, np.ones([n + 1, 1]) / (n + 1))

    # First find minimizer xr on reduced model, then find the 2D point corresponding to xr. Constrained optm.
    A_simplex, b_simplex = Utils.search_simplex_bounds(simplex)
    lb_simplex = np.min(simplex, axis=1)
    ub_simplex = np.max(simplex, axis=1)

    m = n + 1  # The number of constraints which is determined by the number of simplex boundaries.
    assert m == A_simplex.shape[0], 'The No. of simplex constraints is wrong'

    # nF: The number of problem functions in F(x),
    # including the objective function, linear and nonlinear constraints.

    # ObjRow indicates the numer of objective row in F(x).
    ObjRow  = 1

    if n > 1:
        # The first function in F(x) is the objective function, the rest are m simplex constraints.
        # The last part of functions in F(x) is the safe constraints.
        # In high dimension, A_simplex make sure that linear_derivative_A won't be all zero.

        nF = 1 + m + M  # the last 1 is the safe constraint.
        # Since adaptiveK using ( p(x) - f0 ) / e(x), the objective function is nonlinear.
        # The constraints are generated by simplex bounds, all linear.
        Flow = np.hstack((-inf, b_simplex.T[0], np.zeros(M) ))
        Fupp = inf * np.ones(nF)

        # The lower and upper bounds of variables x.
        xlow    = np.copy(lb_simplex) + finest_mesh/2
        xupp    = np.copy(ub_simplex) - finest_mesh/2

        # For the nonlinear components, enter any nonzero value in G to indicate the location
        # of the nonlinear derivatives (in this case, 2).

        # A must be properly defined with the correct derivative values.
        linear_derivative_A    = np.vstack((np.zeros((1, n)), A_simplex, np.zeros((M, n)) ))
        nonlinear_derivative_G = np.vstack((2 * np.ones((1, n)), np.zeros((m, n)), 2 * np.ones((M, n)) ))

    else:
        # For 1D problem, only have 1 objective function, the simplex constraint is defined by x bounds.
        # nF = 1 + M, 1 obj + M safe cons.
        nF = 1 + M + 1

        Flow    = np.hstack((-inf, np.zeros(M), -inf ))
        Fupp    = inf * np.ones(nF)
        xlow    = np.copy(lb_simplex) + finest_mesh/2
        xupp    = np.copy(ub_simplex) - finest_mesh/2

        linear_derivative_A    = np.vstack(( np.zeros((1, n)), np.zeros((M, n)), np.ones((1, n)) ))
        nonlinear_derivative_G = np.vstack(( 2 * np.ones((1 + M, n)), np.zeros((1, n)) ))

    x0      = x.T[0]
    save_opt_for_snopt_ak(n, nF, inter_par, xc, R2, y0, A_simplex, y_safe, L_safe, exist, b, c)

    options = SNOPT_options()
    options.setOption('Infinite bound', inf)
    options.setOption('Verify level', 3)
    options.setOption('Verbose', False)
    options.setOption('Print level', -1)
    options.setOption('Scale option', 2)
    options.setOption('Print frequency', -1)
    options.setOption('Scale option', 2)
    options.setOption('Feasibility tolerance', 1e-5)
    options.setOption('Summary', 'No')

    sol = snopta(dogsobj, n, nF, x0=x0, name='DeltaDOGS_snopt', xlow=xlow, xupp=xupp, Flow=Flow, Fupp=Fupp,
                    ObjRow=ObjRow, A=linear_derivative_A, G=nonlinear_derivative_G, options=options)

    x = sol.x
    y = sol.objective

    return x.reshape(-1, 1), y

def save_opt_folder_path():
    current_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    folder = current_path[:-5]      # -5 comes from the length of '/dogs'
    return folder

def save_opt_for_snopt_ak(n, nF, inter_par, xc, R2, y0, A_simplex, y_safe, L_safe, exist, b, c):
    var_opt = {}
    folder = save_opt_folder_path()
    if inter_par.method == "NPS":
        var_opt['inter_par_method'] = inter_par.method
        var_opt['inter_par_w']      = inter_par.w
        var_opt['inter_par_v']      = inter_par.v
        var_opt['inter_par_xi']     = inter_par.xi
    var_opt['n']  = n
    var_opt['nF'] = nF
    var_opt['xc'] = xc
    var_opt['R2'] = R2
    var_opt['y0'] = y0
    var_opt['A']  = A_simplex
    var_opt['y_safe'] = y_safe
    var_opt['L_safe'] = L_safe
    var_opt['exist']  = exist
    var_opt['b']   = b
    var_opt['c'] = c
    io.savemat(folder + "/opt_info_ak.mat", var_opt)
    return

def adaptivek_search_cost_snopt(x):
    x       = x.reshape(-1, 1)
    folder  = save_opt_folder_path()
    var_opt = io.loadmat(folder + "/opt_info_ak.mat")

    n  = var_opt['n'][0, 0]
    xc = var_opt['xc']
    R2 = var_opt['R2'][0, 0]
    y0 = var_opt['y0'][0, 0]
    nF = var_opt['nF'][0, 0]
    A  = var_opt['A']
    b  = var_opt['b'][0, 0]
    c  = var_opt['c'][0, 0]
    y_safe = var_opt['y_safe']
    L_safe = var_opt['L_safe'][0, 0]
    exist  = var_opt['exist'][0, 0]

    # Initialize the output F.
    F = np.zeros(nF)
    M = y_safe.shape[0]

    method       = var_opt['inter_par_method'][0]
    inter_par    = interpolation.Inter_par(method=method)
    inter_par.w  = var_opt['inter_par_w']
    inter_par.v  = var_opt['inter_par_v']
    inter_par.xi = var_opt['inter_par_xi']

    p  = interpolation.interpolate_val(x, inter_par)
    gp = interpolation.interpolate_grad(x, inter_par)

    if exist == 0:
        e = R2 - np.linalg.norm(x - xc) ** 2
        ge = - 2 * (x - xc)
    else:
        e, ge, gge = constant_snopt_min.discrete_min_uncertainty(x, inter_par.xi, b, c)

    # denominator = (1e-10 if abs(p-y0) < 1e-10 else p - y0)
    # F[0] = - e / denominator
    # DM   = - ge / denominator + e * gp / denominator ** 2
    F[0] = (p - y0) / e
    DM   = gp / e - ge * (p - y0) / e ** 2
    # G1: The gradient of the objective function, the continuous search function.
    G1   = DM.flatten()

    val, idx, x_nn = Utils.mindis(x, inter_par.xi)
    norm2_difference = np.sqrt(np.dot((x - x_nn).T, x - x_nn))
    norm2_difference = ( 1e-15 if norm2_difference < 1e-15 else norm2_difference )

    # G2: Safety function constraint gradient, flattened version, trick is size = M.
    G2 = np.tile((- L_safe * (x - x_nn) / norm2_difference).T[0], M)

    if n > 1:
        # nD data has n+1 simplex bounds.
        F[1 : 1 + (n + 1)] = (np.dot(A, x)).T[0]
        F[1 + (n + 1) : 1 + (n + 1) + M] = y_safe[:, idx] - L_safe * norm2_difference * np.ones(M)

    else:
        F[1 : 1 + M] = y_safe[:, idx] - L_safe * norm2_difference * np.ones(M)
        F[-1] = np.sum(x)

    G = np.hstack((G1, G2))

    return F, G

def dogsobj(status, x, needF, F, needG, G):
    # G is the nonlinear part of the Jacobian
    F, G = adaptivek_search_cost_snopt(x)
    return status, F, G
# =====================================   Interpolant Minimization   ==================================

def interpolation_min_safe_solver(simplex, inter_par, y0, y_safe, L_safe, delta, tol):

    xE = inter_par.xi
    n = xE.shape[0]
    x = np.dot(simplex, np.ones([n + 1, 1]) / (n + 1))
    # First: Determine the estimated safety function values at x ( p(x) < y0 ).
    val, idx, x_nn = Utils.mindis(x, xE)
    safe_estimate_x = y_safe[:, idx] - L_safe * val

    if (safe_estimate_x > 0 ).all():
        # Safety is guaranteed, root finding p(x) along x and x_nn
        f = lambda x: interpolation.interpolate_val(x, inter_par) - y0
        result = 'minimize p(x)'
    else:
        # Safety is not guaranteed, root finding safety function along x and x_nn
        f = lambda x:  y_safe[:, idx] - L_safe * np.linalg.norm(x - x_nn)
        result = 'minimize hat_psi(x)'

    xmin, status = bisection_root_finding(f, x, x_nn, delta, tol)
    ymin = f(xmin)
    return xmin, ymin, result

def bisection_root_finding(f, a, b, delta, tol):
    '''
    Finding the root for the objective function f, could be slightly positive solution due to safety functions.
    Thought about using false position method at the first time, but it seems that falsi method only works for
    1D function.
    :param f    :   Objective function, could be multi-dimensional.
    :param a    :   Center of the circumcircle.
    :param b    :   The closest evaluated data point to a.
    :param delta:   Tolerance of distance of your interval on variable x.
    :param tol  :   Tolerance of objective function values.
    :return     :   The root of f(must be >= 0).
    '''
    num_iter = 1000
    # The hard thing is, you dont know the evaluated data is upper bound or the lower bound.
    if np.linalg.norm(a) < np.linalg.norm(b):
        low_bnd_b = 0
        low_bnd = a
        upp_bnd = b
    else:
        low_bnd_b = 1
        low_bnd = b
        upp_bnd = a
    flow = f(low_bnd)
    fupp = f(upp_bnd)
    eps = delta

    e = np.copy(upp_bnd - low_bnd)

    if ((np.sign(flow) > 0).all() and (np.sign(fupp) > 0).all()) or ((np.sign(flow) < 0).all() and (np.sign(fupp) < 0).all()):
        x = np.copy(b)
        status = 'Error message: Boundaries have the same sign, should not happen in this case.'
    else:
        status = 'Maximum iteration reached.'
        for i in range(num_iter):
            e /= 2
            m = low_bnd + e
            fm = f(m)
            if np.max(e) < delta:
                status = 'Required error of subinterval bound achieved'
                break
            elif np.max(np.abs(fm)) < tol:
                status = 'The tolerance on the objective function value is reached.'
                break
            else:
                if (np.sign(fm) > 0 + tol).all():
                    # mid point is all positive, set the bound close to the evaluated data to be the mid point
                    if low_bnd_b == 1:
                        low_bnd = np.copy(m)
                        flow = np.copy(fm)
                    else:
                        upp_bnd = np.copy(m)
                        fupp = np.copy(fm)
                else:
                    # mid point has negative part, set bound close to the circumcenter to be the mid point
                    if low_bnd_b == 1:
                        upp_bnd = np.copy(m)
                        fupp = np.copy(fm)
                    else:
                        low_bnd = np.copy(m)
                        fupp = np.copy(fm)
        if (np.sign(fm) > 0 + tol).all():  # mid point satisfy the safety functions
            x = np.copy(m)
        else:  # mid point does not satisfy the safety functions, using the bound close to the evaluated data points.
            if low_bnd_b == 1:
                x = np.copy(low_bnd)
            else:
                x = np.copy(upp_bnd)
    return x, status


from __future__ import annotations

from typing import Any
from typing import Optional

from . import ext
from .._typing import _OnConflictConstraintT
from .._typing import _OnConflictIndexElementsT
from .._typing import _OnConflictIndexWhereT
from .._typing import _OnConflictSetT
from .._typing import _OnConflictWhereT
from ... import util
from ...sql import coercions
from ...sql import roles
from ...sql import schema
from ...sql._typing import _DMLTableArgument
from ...sql.base import _exclusive_against
from ...sql.base import _generative
from ...sql.base import ColumnCollection
from ...sql.base import ReadOnlyColumnCollection
from ...sql.dml import Insert as StandardInsert
from ...sql.elements import ClauseElement
from ...sql.elements import KeyedColumnElement
from ...sql.expression import alias
from ...util.typing import Self

__all__ = ("Insert", "insert")

def insert(table: _DMLTableArgument) -> Insert:
    """Construct a PostgreSQL-specific variant :class:`_postgresql.Insert`
    construct.

    .. container:: inherited_member

        The :func:`sqlalchemy.dialects.postgresql.insert` function creates
        a :class:`sqlalchemy.dialects.postgresql.Insert`.  This class is based
        on the dialect-agnostic :class:`_sql.Insert` construct which may
        be constructed using the :func:`_sql.insert` function in
        SQLAlchemy Core.

    The :class:`_postgresql.Insert` construct includes additional methods
    :meth:`_postgresql.Insert.on_conflict_do_update`,
    :meth:`_postgresql.Insert.on_conflict_do_nothing`.

    """
    return Insert(table)

class Insert(StandardInsert):
    """PostgreSQL-specific implementation of INSERT.

    Adds methods for PG-specific syntaxes such as ON CONFLICT.

    The :class:`_postgresql.Insert` object is created using the
    :func:`sqlalchemy.dialects.postgresql.insert` function.

    """

    stringify_dialect = "postgresql"
    inherit_cache = False

    @util.memoized_property
    def excluded(
        self,
    ) -> ReadOnlyColumnCollection[str, KeyedColumnElement[Any]]:
        """Provide the ``excluded`` namespace for an ON CONFLICT statement

        PG's ON CONFLICT clause allows reference to the row that would
        be inserted, known as ``excluded``.  This attribute provides
        all columns in this row to be referenceable.

        .. tip::  The :attr:`_postgresql.Insert.excluded` attribute is an
            instance of :class:`_expression.ColumnCollection`, which provides
            an interface the same as that of the :attr:`_schema.Table.c`
            collection described at :ref:`metadata_tables_and_columns`.
            With this collection, ordinary names are accessible like attributes
            (e.g. ``stmt.excluded.some_column``), but special names and
            dictionary method names should be accessed using indexed access,
            such as ``stmt.excluded["column name"]`` or
            ``stmt.excluded["values"]``.   See the docstring for
            :class:`_expression.ColumnCollection` for further examples.

        .. seealso::

            :ref:`postgresql_insert_on_conflict` - example of how
            to use :attr:`_expression.Insert.excluded`

        """
        return alias(self.table, name="excluded").columns

    _on_conflict_exclusive = _exclusive_against(
        "_post_values_clause",
        msgs={
            "_post_values_clause": "This Insert construct already has "
            "an ON CONFLICT clause established"
        },
    )

    @_generative
    @_on_conflict_exclusive
    def on_conflict_do_update(
        self,
        constraint: _OnConflictConstraintT = None,
        index_elements: _OnConflictIndexElementsT = None,
        index_where: _OnConflictIndexWhereT = None,
        set_: _OnConflictSetT = None,
        where: _OnConflictWhereT = None,
    ) -> Self:
        r"""
        Specifies a DO UPDATE SET action for ON CONFLICT clause.

        Either the ``constraint`` or ``index_elements`` argument is
        required, but only one of these can be specified.

        :param constraint:
         The name of a unique or exclusion constraint on the table,
         or the constraint object itself if it has a .name attribute.

        :param index_elements:
         A sequence consisting of string column names, :class:`_schema.Column`
         objects, or other column expression objects that will be used
         to infer a target index.

        :param index_where:
         Additional WHERE criterion that can be used to infer a
         conditional target index.

        :param set\_:
         A dictionary or other mapping object
         where the keys are either names of columns in the target table,
         or :class:`_schema.Column` objects or other ORM-mapped columns
         matching that of the target table, and expressions or literals
         as values, specifying the ``SET`` actions to take.

         .. versionadded:: 1.4 The
            :paramref:`_postgresql.Insert.on_conflict_do_update.set_`
            parameter supports :class:`_schema.Column` objects from the target
            :class:`_schema.Table` as keys.

         .. warning:: This dictionary does **not** take into account
            Python-specified default UPDATE values or generation functions,
            e.g. those specified using :paramref:`_schema.Column.onupdate`.
            These values will not be exercised for an ON CONFLICT style of
            UPDATE, unless they are manually specified in the
            :paramref:`.Insert.on_conflict_do_update.set_` dictionary.

        :param where:
         Optional argument. If present, can be a literal SQL
         string or an acceptable expression for a ``WHERE`` clause
         that restricts the rows affected by ``DO UPDATE SET``. Rows
         not meeting the ``WHERE`` condition will not be updated
         (effectively a ``DO NOTHING`` for those rows).

        .. seealso::

            :ref:`postgresql_insert_on_conflict`

        """
        self._post_values_clause = OnConflictDoUpdate(
            constraint, index_elements, index_where, set_, where
        )
        return self

    @_generative
    @_on_conflict_exclusive
    def on_conflict_do_nothing(
        self,
        constraint: _OnConflictConstraintT = None,
        index_elements: _OnConflictIndexElementsT = None,
        index_where: _OnConflictIndexWhereT = None,
    ) -> Self:
        """
        Specifies a DO NOTHING action for ON CONFLICT clause.

        The ``constraint`` and ``index_elements`` arguments
        are optional, but only one of these can be specified.

        :param constraint:
         The name of a unique or exclusion constraint on the table,
         or the constraint object itself if it has a .name attribute.

        :param index_elements:
         A sequence consisting of string column names, :class:`_schema.Column`
         objects, or other column expression objects that will be used
         to infer a target index.

        :param index_where:
         Additional WHERE criterion that can be used to infer a
         conditional target index.

        .. seealso::

            :ref:`postgresql_insert_on_conflict`

        """
        self._post_values_clause = OnConflictDoNothing(
            constraint, index_elements, index_where
        )
        return self

class OnConflictClause(ClauseElement):
    stringify_dialect = "postgresql"

    constraint_target: Optional[str]
    inferred_target_elements: _OnConflictIndexElementsT
    inferred_target_whereclause: _OnConflictIndexWhereT

    def __init__(
        self,
        constraint: _OnConflictConstraintT = None,
        index_elements: _OnConflictIndexElementsT = None,
        index_where: _OnConflictIndexWhereT = None,
    ):
        if constraint is not None:
            if not isinstance(constraint, str) and isinstance(
                constraint,
                (schema.Constraint, ext.ExcludeConstraint),
            ):
                constraint = getattr(constraint, "name") or constraint

        if constraint is not None:
            if index_elements is not None:
                raise ValueError(
                    "'constraint' and 'index_elements' are mutually exclusive"
                )

            if isinstance(constraint, str):
                self.constraint_target = constraint
                self.inferred_target_elements = None
                self.inferred_target_whereclause = None
            elif isinstance(constraint, schema.Index):
                index_elements = constraint.expressions
                index_where = constraint.dialect_options["postgresql"].get(
                    "where"
                )
            elif isinstance(constraint, ext.ExcludeConstraint):
                index_elements = constraint.columns
                index_where = constraint.where
            else:
                index_elements = constraint.columns
                index_where = constraint.dialect_options["postgresql"].get(
                    "where"
                )

        if index_elements is not None:
            self.constraint_target = None
            self.inferred_target_elements = index_elements
            self.inferred_target_whereclause = index_where
        elif constraint is None:
            self.constraint_target = (
                self.inferred_target_elements
            ) = self.inferred_target_whereclause = None

class OnConflictDoNothing(OnConflictClause):
    __visit_name__ = "on_conflict_do_nothing"

class OnConflictDoUpdate(OnConflictClause):
    __visit_name__ = "on_conflict_do_update"

    def __init__(
        self,
        constraint: _OnConflictConstraintT = None,
        index_elements: _OnConflictIndexElementsT = None,
        index_where: _OnConflictIndexWhereT = None,
        set_: _OnConflictSetT = None,
        where: _OnConflictWhereT = None,
    ):
        super().__init__(
            constraint=constraint,
            index_elements=index_elements,
            index_where=index_where,
        )

        if (
            self.inferred_target_elements is None
            and self.constraint_target is None
        ):
            raise ValueError(
                "Either constraint or index_elements, "
                "but not both, must be specified unless DO NOTHING"
            )

        if isinstance(set_, dict):
            if not set_:
                raise ValueError("set parameter dictionary must not be empty")
        elif isinstance(set_, ColumnCollection):
            set_ = dict(set_)
        else:
            raise ValueError(
                "set parameter must be a non-empty dictionary "
                "or a ColumnCollection such as the `.c.` collection "
                "of a Table object"
            )
        self.update_values_to_set = [
            (coercions.expect(roles.DMLColumnRole, key), value)
            for key, value in set_.items()
        ]
        self.update_whereclause = where


import ipaddress
import re
import glob

def Return_IP_adr(str):
    if re.match("^ ip address ([0-9.]+) ([0-9.]+)$", str):
        r = re.match("^ ip address ([0-9.]+) ([0-9.]+)$", str)
        return ipaddress.IPv4Network((r.group(1), r.group(2)),strict = False)
    else:
        return "None"

list_files = glob.glob("C:\\Users\\nv.solomennikova\\Documents\\pythonProject\\p4ne\\Lab1.5\\config_files\\*.txt")
list_result = []
for fl in list_files:
    with open (fl) as f:
        for s in f:
            res = Return_IP_adr(s)
            if res != "None":
                list_result.append(res)

print((list(set(list_result))))

'''
r = re.match("^ ip address ([0-9.]+) ([0-9.]+)$", " ip address 192.168.1.1 255.255.255.0")
print(r.group(2))
res = ipaddress.IPv4Network((r.group(1), r.group(2)),strict = False)

'''