#!/usr/bin/env python
from __future__ import print_function
from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand
import io
import os

here = os.path.abspath(os.path.dirname(__file__))

def read_version(version_filepath):
    import re

    verstrline = open(version_filepath, "rt").read()
    VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]"
    mo = re.search(VSRE, verstrline, re.M)
    return mo.group(1)

#
def read(*filenames, **kwargs):
    encoding = kwargs.get("encoding", "utf-8")
    sep = kwargs.get("sep", "\n")
    buf = []
    for filename in filenames:
        with io.open(filename, encoding=encoding) as f:
            buf.append(f.read())
    return sep.join(buf)

long_description = read("README.md")

setup(
    name="Instarecon",
    version=read_version("src/_version.py"),
    url="http://github.com/vergl4s/instarecon/",
    license="MIT License",
    author="Luis Teixeira",
    install_requires=[
        "dnspython>=1.14.0",
        "ipaddr>=2.1.11",
        "ipaddress>=1.0.16",
        "ipwhois>=0.13.0",
        "pythonwhois>=2.4.3",
        "requests>=2.10.0",
        "shodan>=1.5.3",
    ],
    scripts=["scripts/instarecon.py"],
    author_email="luis@teix.co",
    description="Automated basic digital reconnaissance",
    packages=find_packages("."),
    include_package_data=True,
    classifiers=[
        "Development Status :: 4 - Beta",
        "Natural Language :: English",
        "Programming Language :: Python :: 2.7",
        "Intended Audience :: Developers, Hackers",
        "License :: OSI Approved :: MIT License",
    ],
    test_suite="tests",
)


from collections import deque
cardeck = deque()

n = int(input())
for i in range(1,n+1):
    cardeck.append(i)
while True:
    if len(cardeck) == 1:
        break
    cardeck.popleft()
    k = cardeck.popleft()
    cardeck.append(k)
print(cardeck[0])


import pandas as pd
import numpy as np
import random
from sklearn import svm
from PIL import Image

# download present points of species
present = pd.read_csv('data/present_points.csv')
present = pd.DataFrame(present)
present_array = np.array([present['latitude'],present['longitude']])

y_array = np.array(present['latitude'].to_list())
x_array = np.array(present['longitude'].to_list())

# download worldclim data
worldclim = np.load('data/worldclim_new.npy')

def get_values_from_matrix(worldclim, pa, min_x, min_y):
    # get worldclim data to selected points
    new_matrix = []
    min_x_new = 21600 + round(min_x / 180 * 21600)
    min_y_new = 10800 - round(min_y / 90 * 10800)
    for i in range(len(pa[0])):
        x_new = (21600 + round(pa[1, i] / 180 * 21600)) - min_x_new
        y_new = (10800 - round(pa[0, i] / 90 * 10800)) - min_y_new
        new_matrix.append(worldclim[:, y_new, x_new])
    return new_matrix

def get_values(worldclim, pa, min_x, min_y):
    # get worldclim data to selected point
    new_matrix = []
    min_x_new = 21600 + round(min_x / 180 * 21600)
    min_y_new = 10800 - round(min_y / 90 * 10800)
    x_new = (21600 + round(pa[0] / 180 * 21600)) - min_x_new
    y_new = (10800 - round(pa[1] / 90 * 10800)) - min_y_new
    new_matrix.append(worldclim[:, y_new, x_new])
    return new_matrix

# get wordclim data for all presented points
x_pres = get_values_from_matrix(worldclim, present_array, 55, 47)

# define and train OCSVM on present points and worldclim data
clf = svm.OneClassSVM(nu=0.3, kernel="rbf", gamma=0.05)
clf.fit(x_pres)

# create a map of OCSVM prediction
wordclim_new = worldclim.T.reshape((3960000, 19))
prediction = clf.predict(wordclim_new)
prediction = np.array([1 if x == 1 else 0.5 for x in prediction]).reshape((3000, 1320)).T

new_matrix = (prediction * 255).astype(np.uint8)
img = Image.fromarray(new_matrix, mode='L')
img.save('pseudo_absence_rsep.jpg')

def find_random(x1, x2, y1, y2):  # RSEP pseudo_absence selecting strategy using OCSVM prediction
    random_x = random.uniform(x1, x2)
    random_y = random.uniform(y1, y2)
    random_array = np.array([random_x, random_y])
    random_data = get_values(worldclim, random_array, 55, 47)
    detector = clf.predict(random_data)
    if detector != -1:
        while detector != -1:
            random_x = random.uniform(x1, x2)
            random_y = random.uniform(y1, y2)
            random_array = np.array([random_x, random_y])
            random_data = get_values(worldclim, random_array, 55, 47)
            detector = clf.predict(random_data)
    else:
        random_x, random_y = random_x, random_y
    return random_x, random_y

def random_float_number(x1, x2, y1, y2, array_x, array_y):  # RSEB pseudo_absence selecting strategy
    random_x = random.uniform(x1, x2)
    random_y = random.uniform(y1, y2)
    if min(abs(array_x-random_x)) <= 0.0833 and min(abs(array_y-random_y)) <= 0.0833:  # set 10 km buffer zone
        while min(abs(array_x-random_x)) <= 0.0833 and min(abs(array_y-random_y)) <= 0.0833:
            random_x = random.uniform(x1, x2)
            random_y = random.uniform(y1, y2)
    else:
        random_x = random_x
        random_y = random_y
    return random_x, random_y

def random_coordinates(extent):  # RS pseudo_absence selecting strategy
    random_x = random.uniform(extent[0], extent[1])
    random_y = random.uniform(extent[2], extent[3])
    return random_x, random_y

extent = [65, 74, 43, 37]
x_pseudo_rs=[]
y_pseudo_rs=[]
x_pseudo_rseb=[]
y_pseudo_rseb=[]
x_pseudo_rsep=[]
y_pseudo_rsep=[]

# RSEP pseudo_absence sampling
for i in range(len(x_pres)*10):
    x_new, y_new = find_random(65.000, 74.000, 37.000, 43.000)
    x_pseudo_rsep.append(x_new)
    y_pseudo_rsep.append(y_new)

# save RSEP sampling in .csv file
species = ['pseudo_absence' for i in range(len(x_pseudo_rsep))]
dict_rsep = {
    'species': species,
    'longitude': x_pseudo_rsep,
    'latitude': y_pseudo_rsep
}
df_rsep = pd.DataFrame(dict_rsep)
df_rsep.to_csv('unused/pseudo_absence_rsep.csv', index=False)

# RSEB pseudo_absence sampling
for i in range(len(x_pres)*10):
    x_new, y_new = random_float_number(65.000, 74.000, 37.000, 43.000, x_array, y_array)
    x_pseudo_rseb.append(x_new)
    y_pseudo_rseb.append(y_new)

# save RSEB sampling in .csv file
dict_rseb = {
    'species': species,
    'longitude': x_pseudo_rseb,
    'latitude': y_pseudo_rseb
}
df_rseb = pd.DataFrame(dict_rseb)
df_rseb.to_csv('unused/pseudo_absence_rseb.csv', index=False)

# RS pseudo_absence sampling
for i in range(len(x_pres)*10):
    x_new, y_new = random_coordinates(extent)
    x_pseudo_rs.append(x_new)
    y_pseudo_rs.append(y_new)

# save RS sampling in .csv file
dict_rs = {
    'species': species,
    'longitude': x_pseudo_rs,
    'latitude': y_pseudo_rs
}
df_rs = pd.DataFrame(dict_rs)
df_rs.to_csv('unused/pseudo_absence_rs.csv', index=False)
print('saved')


#!/usr/bin/env python
import time
import serial
import random
import binascii
import pickle

tty="/dev/ttyAMA0"
baud=57600

config = []

config.append("sys reset")

#change these to values given
config.append("mac set devaddr 260114C8")
config.append("mac set deveui 006117063BA98093")
config.append("mac set appeui 70B3D57EF0005B5B")
config.append("mac set appskey XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
config.append("mac set nwkskey XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")

config.append("mac join abp")

config.append("sys set pindig GPIO0 1")

def send(data, p):
	p.write(data+"\x0d\x0a")
	data.rstrip()
	print(data)
	time.sleep(0.1)
	rdata=p.readline()
	rdata=rdata[:-1]
	print (rdata)

p = serial.Serial(tty , baud)

time.sleep(1)

for c in config:
    send(c, p)

#replace this with the sensor data.
#order is light, red, green, blue, x,y,z, heading, temp, pressure

user_data =[898,166,147,135,-0.0025634765625,0.02557373046875,1.0029296875,137.28,28.2777963716,100794.557256]

data = binascii.hexlify(pickle.dumps(user_data)).upper()
print(data)

send("mac tx cnf 1 {}".format(data), p)

time.sleep(1)
rdata=p.readline()
rdata=rdata[:-1]
print (rdata)
time.sleep(5)
rdata=p.readline()
rdata=rdata[:-1]
print (rdata)


import argparse
import glob
import os

import cv2
import h5py
import numpy as np
import tqdm

import math
from functools import partial

import matplotlib.pyplot as plt

def readyuv420(filename, bitdepth, W, H, startframe,
               totalframe, show=False):
    # 从第startframe（含）开始读（0-based），共读totalframe帧

    uv_H = H // 2
    uv_W = W // 2

    if bitdepth == 8:
        Y = np.zeros((totalframe, H, W), np.uint8)
        U = np.zeros((totalframe, uv_H, uv_W), np.uint8)
        V = np.zeros((totalframe, uv_H, uv_W), np.uint8)
    elif bitdepth == 10:
        Y = np.zeros((totalframe, H, W), np.uint16)
        U = np.zeros((totalframe, uv_H, uv_W), np.uint16)
        V = np.zeros((totalframe, uv_H, uv_W), np.uint16)

    plt.ion()

    bytes2num = partial(int.from_bytes, byteorder='little', signed=False)

    bytesPerPixel = math.ceil(bitdepth / 8)
    seekPixels = startframe * H * W * 3 // 2
    fp = open(filename, 'rb')
    fp.seek(bytesPerPixel * seekPixels)

    for i in range(totalframe):

        for m in range(H):
            for n in range(W):
                if bitdepth == 8:
                    pel = bytes2num(fp.read(1))
                    Y[i, m, n] = np.uint8(pel)
                elif bitdepth == 10:
                    pel = bytes2num(fp.read(2))
                    Y[i, m, n] = np.uint16(pel)

        for m in range(uv_H):
            for n in range(uv_W):
                if bitdepth == 8:
                    pel = bytes2num(fp.read(1))
                    U[i, m, n] = np.uint8(pel)
                elif bitdepth == 10:
                    pel = bytes2num(fp.read(2))
                    U[i, m, n] = np.uint16(pel)

        for m in range(uv_H):
            for n in range(uv_W):
                if bitdepth == 8:
                    pel = bytes2num(fp.read(1))
                    V[i, m, n] = np.uint8(pel)
                elif bitdepth == 10:
                    pel = bytes2num(fp.read(2))
                    V[i, m, n] = np.uint16(pel)

        if show:
            print(i)
            plt.subplot(131)
            plt.imshow(Y[i, :, :], cmap='gray')
            plt.subplot(132)
            plt.imshow(U[i, :, :], cmap='gray')
            plt.subplot(133)
            plt.imshow(V[i, :, :], cmap='gray')
            plt.show()
            plt.pause(100)
            # plt.pause(0.001)

    if totalframe == 1:
        return Y[0], U[0], V[0]
    else:
        return Y, U, V

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input_path', type=str)
    parser.add_argument('-o', '--output_path', type=str)
    parser.add_argument('-a', '--area_portion', type=float, default=0.5)
    parser.add_argument('-b', '--max_blocks', type=int, default=2000)
    parser.add_argument('-e', '--ext_bound', type=bool, default=False)
    parser.add_argument('-d', '--bit_depth', type=int, default=10)
    parser.add_argument('-nm', '--normalize', type=bool, default=True)
    args = parser.parse_args()

    qps = [22, 27, 32, 37]
    classes = [0, 0, 1, 2, 3, 4, 5, 6]
    block_shapes = [(4, 4), (8, 8), (16, 16)]
    keys = ['org_chroma', 'rec_boundary_y', 'rec_boundary_uv', 'rec_luma']
    path_org = args.input_path + '/DIV2K_yuv'
    path_rec = args.input_path + '/rec'

    output_path = args.output_path + '/%.2f-%d-%d' % (args.area_portion, args.max_blocks, int(args.ext_bound))
    if not os.path.exists(args.output_path): os.mkdir(args.output_path)
    if not os.path.exists(output_path): os.mkdir(output_path)

    with open(output_path + '/config.txt', 'w') as logs:
        logs.write('area portion: %.2f\n' % args.area_portion)
        logs.write('max_blocks: %d\n' % args.max_blocks)
        logs.write('ext_bound: %d\n' % int(args.ext_bound))
        logs.write('bit depth: %d\n' % args.bit_depth)
        logs.write('normalize: %d\n' % int(args.normalize))

        for mode in ['train', 'val']:
            print(mode)
            input_path = os.path.join(path_org, mode)
            input_path_rec = os.path.join(path_rec, mode)
            mode_path = os.path.join(output_path, mode)
            if not os.path.exists(mode_path): os.mkdir(mode_path)

            output = {}
            for shape in block_shapes:
                boundary_size_y = 2 * sum(shape) + 1 if not args.ext_bound \
                    else ((4 * shape[0]) + (4 * shape[1]) + 1)
                boundary_size_uv = 2 * (sum(shape) + 1) if not args.ext_bound \
                    else 2 * ((2 * shape[0]) + (2 * shape[1]) + 1)
                hf = h5py.File('%s/%dx%d.h5' % (mode_path, shape[0], shape[1]), 'w')
                hf.create_dataset('rec_luma', (1, shape[0] * 2, shape[1] * 2, 1), maxshape=(None, shape[0] * 2, shape[1] * 2, 1))
                hf.create_dataset('rec_boundary_y', (1, boundary_size_y), maxshape=(None, boundary_size_y))
                hf.create_dataset('rec_boundary_uv', (1, boundary_size_uv), maxshape=(None, boundary_size_uv))
                hf.create_dataset('org_chroma', (1, shape[0], shape[1], 2), maxshape=(None, shape[0], shape[1], 2))
                output["%dx%d" % shape] = hf

            blocks = {"%dx%d" % b: 0 for b in block_shapes}
            files = glob.glob(os.path.join(input_path, '0/*'))
            files.sort()
            start = int(files[0].split('/')[-1].split('_')[0])
            for file_id in tqdm.tqdm(range(start, start + len(files))):
                cat = np.random.choice(classes, 1)
                file_path = glob.glob(os.path.join(input_path, '{:d}' + '/{:04d}' + '*.yuv').format(cat[0], file_id))
                file_name = file_path[0].split('/')[-1]
                yuv = "{}/{:d}/{}".format(input_path, cat[0], file_name)
                name = file_name.split('.')[0]
                size = name.split('_')[1]
                width = int(size.split('x')[0])//2
                height = int(size.split('x')[1])//2
                y, u, v = readyuv420(yuv, 10, width * 2, height * 2, 0, 1, False)
                recyuv = os.path.join(input_path_rec, '{:d}', name).format(cat[0])
                #for qp in qps:
                qp = np.random.choice(qps, 1)
                recyuv_qp = os.path.join(recyuv, '{}.yuv').format(qp[0])
                recy, recu, recv = readyuv420(recyuv_qp, 10, width * 2, height * 2, 0, 1, False)
                y = y.astype(np.float32)
                u = u.astype(np.float32)
                v = v.astype(np.float32)
                recy = recy.astype(np.float32)
                recu = recu.astype(np.float32)
                recv = recv.astype(np.float32)
                if args.normalize:
                    y /= (2 ** args.bit_depth - 1)
                    u /= (2 ** args.bit_depth - 1)
                    v /= (2 ** args.bit_depth - 1)
                    recy /= (2 ** args.bit_depth - 1)
                    recu /= (2 ** args.bit_depth - 1)
                    recv /= (2 ** args.bit_depth - 1)

                for shape in block_shapes:
                    portion = min(int((height * width * args.area_portion) /
                                      (shape[0] * shape[1])), args.max_blocks)
                    blocks["%dx%d" % shape] += portion
                    lm = 1 if not args.ext_bound else 2
                    xx, yy = np.mgrid[1:height - lm * shape[0]:shape[0],
                             1:width - lm * shape[1]:shape[1]]
                    positions = np.vstack([xx.ravel(), yy.ravel()]).T
                    positions = positions[np.random.choice(np.arange(len(positions)), portion)]
                    data = output["%dx%d" % shape]
                    info = {'rec_boundary_y': [], 'rec_boundary_uv': [], 'rec_luma': [], 'org_chroma': []}
                    for p in positions:
                        if not args.ext_bound:
                            rows_y = np.append(np.array([p[0] * 2 - 1] * (shape[1] * 2 + 1)),
                                             np.arange(p[0] * 2, p[0] * 2 + shape[0] * 2))
                            cols_y = np.append(np.arange(p[1] * 2, p[1] * 2 + shape[1] * 2),
                                             np.array([p[1] * 2 - 1] * (shape[0] * 2 + 1)))
                            rows_uv = np.append(np.array([p[0] - 1] * (shape[1] + 1)),
                                             np.arange(p[0], p[0] + shape[0]))
                            cols_uv = np.append(np.arange(p[1], p[1] + shape[1]),
                                             np.array([p[1] - 1] * (shape[0] + 1)))
                        else:
                            rows = np.append(np.array([p[0] - 1] * (1 + shape[1] * 2)),
                                             np.arange(p[0], p[0] + shape[0] * 2))
                            cols = np.append(np.arange(p[1], p[1] + shape[1] * 2),
                                             np.array([p[1] - 1] * (1 + shape[0] * 2)))
                        # rec boundaries_y
                        values = np.append([], [ch for ch in recy[rows_y, cols_y].T])
                        info['rec_boundary_y'].append(values)
                        # rec boundaries_uv
                        recuv = np.concatenate((np.expand_dims(recu, -1), np.expand_dims(recv, -1)), axis=2)
                        values = np.append([], [ch for ch in recuv[rows_uv, cols_uv, :].T])
                        info['rec_boundary_uv'].append(values)
                        # rec luma
                        values = np.expand_dims(recy[p[0] * 2:p[0] * 2 + shape[0] * 2, p[1] * 2:p[1] * 2 + shape[1] * 2], -1)
                        info['rec_luma'].append(values)
                        # org chroma
                        values_u = np.expand_dims(u[p[0]:p[0] + shape[0], p[1]:p[1] + shape[1]], -1)
                        values_v = np.expand_dims(v[p[0]:p[0] + shape[0], p[1]:p[1] + shape[1]], -1)
                        values = np.concatenate((values_u, values_v), axis=2)
                        info['org_chroma'].append(values)

                    for key in keys:
                        data[key].resize((data[key].shape[0] + len(positions) - 1), axis=0)
                        data[key][-len(positions):] = info[key]
                        if file_id - start < len(files) - 1:
                            data[key].resize((data[key].shape[0] + 1), axis=0)

            logs.write('---\n%s\n' % mode)
            for shape in block_shapes:
                b = "%dx%d" % shape
                logs.write("%s: %d\n" % (b, blocks[b]))
                output[b].close()


import csv

from io import StringIO
from typing import Dict, Iterator, List, Tuple, Union

DEFAULT_HASH_MAP_SIZE: int = 64

class HashMap:
    _size: int = None

    _map: List[List[Tuple[str, str]]] = None
    _length: int = None

    @property
    def size(self) -> int:
        return self._size

    def __init__(self, size: int):
        if size < 1:
            raise ValueError("Size must be greater than 0")

        self._size = size
        self._initialize_map()

    def _initialize_map(self) -> None:
        self._map = [None] * self._size
        self._length = 0

    def get(self, key: str, default: str = None) -> Union[str, None]:
        try:
            return self[key]

        except KeyError:
            return default

    def pop(self, key: str) -> str:
        value = self[key]

        del self[key]

        return value

    def keys(self) -> Iterator[str]:
        for item in self.items():
            yield item[0]

    def items(self) -> Iterator[Tuple[str, str]]:
        for bucket in self._map:
            if bucket is not None:
                for item in bucket:
                    yield item

    def values(self) -> Iterator[str]:
        for item in self.items():
            yield item[1]

    def from_csv(self, csv_string: str) -> None:
        """
        Loads the key-value pairs from a CSV format string into the hash map.

        Example:
            >>> hash_map = HashMap(16)
            >>> hash_map.from_csv('a,1\\nb,2\\nc,3\\n')
            >>> hash_map
            HashMap(16): {'a': '1', 'b': '2', 'c': '3'}
            >>> hash_map.from_csv('d,4\\nb,5\\n')
            >>> hash_map
            HashMap(16): {'a': '1', 'b': '5', 'c': '3', 'd': '4'}
        """

        str_io: StringIO = StringIO(csv_string)

        reader = csv.reader(str_io)
        for key, value in reader:
            self[key] = value

    def to_csv(self) -> str:
        """
        Returns the key-value pairs from the hash map into a CSV format string.

        Example:
            >>> hash_map = HashMap(16)
            >>> hash_map['a'] = '1'
            >>> hash_map['b'] = '2'
            >>> hash_map['c'] = '3'
            >>> hash_map
            HashMap(16): {'a': '1', 'b': '2', 'c': '3'}
            >>> hash_map.to_csv()
            'a,1\\nb,2\\nc,3\\n'
        """

        str_io: StringIO = StringIO()

        writer = csv.writer(str_io)
        writer.writerows(self.items())

        return str_io.getvalue()

    def to_dict(self) -> Dict[str, str]:
        return dict(self.items())

    def clear(self) -> None:
        self._initialize_map()

    def __getitem__(self, key: str) -> str:
        hashed_key: int = hash(key) % self._size
        bucket: List[Tuple[str, str]] = self._map[hashed_key]

        if bucket is None:
            raise KeyError(key)

        for item in bucket:
            if item[0] == key:
                return item[1]

        raise KeyError(key)

    def __setitem__(self, key: str, value: str) -> None:
        hashed_key: int = hash(key) % self._size
        bucket: List[Tuple[str, str]] = self._map[hashed_key]

        if bucket is None:
            self._map[hashed_key] = [(key, value)]
            self._length += 1

            return

        for index, item in enumerate(bucket):
            if item[0] == key:
                bucket[index] = (key, value)

                return

        bucket.append((key, value))
        self._length += 1

    def __delitem__(self, key: str) -> None:
        hashed_key: int = hash(key) % self._size
        bucket: List[Tuple[str, str]] = self._map[hashed_key]

        if bucket is None:
            raise KeyError(key)

        for index, item in enumerate(bucket):
            if item[0] == key:
                del bucket[index]
                self._length -= 1

                if len(bucket) == 0:
                    self._map[hashed_key] = None

                return

        raise KeyError(key)

    def __bool__(self) -> bool:
        return bool(self._length)

    def __len__(self) -> int:
        return self._length

    def __iter__(self) -> Iterator[str]:
        return self.values()

    def __contains__(self, key: str) -> bool:
        try:
            self[key]

        except KeyError:
            return False

        return True

    def __str__(self) -> str:
        values: List[str] = []

        for key, value in self.items():
            values.append(f"{repr(key)}: {repr(value)}")

        return f"{{{', '.join(values)}}}"

    def __repr__(self) -> str:
        return f"HashMap({self._size}): {self}"


import array
import itertools
import numpy as np
import math
import threading
from ola.ClientWrapper import ClientWrapper

# globals for timing, script state storage
TICK_INTERVAL = 33  # in ms - so display at approximately 30 fps
TIME_ELAPSED = 0.0  # time in ms since we last displaed begclr
                    # when TIME_ELAPSED is first >= delta_t/2.0,
                    # we display endclr (and begin to cycle back)

def DmxSent(state):
    if not state.Succeeded():
        sys.exit(1)

def SendDMXFrame():
    # calculate values to display

    # figure out weighting for each color during this frame.
    # weighting elements should sum to 1.0 in this script
    # (if they didn't, color intensity as well as hue would change with time)
    global TIME_ELAPSED
    if TIME_ELAPSED < half_width:
        weighting = np.array([(half_width-TIME_ELAPSED)/half_width, TIME_ELAPSED/half_width])
    else:
        since_switch = TIME_ELAPSED - half_width
        weighting = np.array([since_switch/half_width, (half_width-since_switch)/half_width])

    # calculate color to display
    clr = weighting.dot(colors)

    datas = []

    # prepare DMX512 serial data
    for rng in rngs:
        data = array.array('B', [0] * 512)
        for chann in xrange(rng[1], rng[2] - 3, 3):
            data[chann+0] = int(clr[0])
            data[chann+1] = int(clr[1])
            data[chann+2] = int(clr[2])
        datas.append(data)

    # display frame
    for rng, data in itertools.izip(rngs, datas):
        wrapper.Client().SendDmx(rng[0], data, DmxSent)

    # store relevant state for next frame calculation
    TIME_ELAPSED = math.fmod((TIME_ELAPSED + TICK_INTERVAL), period)

    wrapper.AddEvent(TICK_INTERVAL, SendDMXFrame)

def squid(dmxrngs, begclr, endclr, delta_t):
    # arrange beginning and ending RGB colors into a matrix
    # which looks like [begclr,
    #                   endclr].
    global colors
    colors = np.vstack((begclr, endclr))

    global period
    period = delta_t

    global half_width
    half_width = period/2.0

    global rngs
    rngs = dmxrngs

    global wrapper
    wrapper = ClientWrapper()

    wrapper.AddEvent(22, SendDMXFrame)
    wrapper.Run()


# -*- coding: utf-8 -*-
"""
Created on Thu Dec  6 02:42:08 2018

@author: AYSE
"""

#1.Kutuphaneler
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import r2_score

# Veri yukleme
veriler= pd.read_csv("maaslar.csv")

#eğitimseviyes=x, maaslar=y olarak bölüyoruz
#dataframe slice
x=veriler.iloc[:,1:2]
y=veriler.iloc[:,2:]
#numpy array dönüşümü
X=x.values
Y=y.values
#linear regression
#doğrusal model oluşturma
from sklearn.linear_model import LinearRegression
lin_reg=LinearRegression()
lin_reg.fit(X,Y)

#polynomial regression
#doğrusal olmayan(nonlinear) model oluşturma-2.derecen polinom
from sklearn.preprocessing import PolynomialFeatures

poly_reg=PolynomialFeatures(degree=2)
x_poly=poly_reg.fit_transform(X)
lin_reg2=LinearRegression()
lin_reg2.fit(x_poly,y)

#4.dereceden polinom
poly_reg3=PolynomialFeatures(degree=4)
x_poly3=poly_reg3.fit_transform(X)
lin_reg3=LinearRegression()
lin_reg3.fit(x_poly3,y)

#Görselleştirme
plt.scatter(X,Y,color='red')
#linear regression gibi kullanıyoruz fakat predict ederken herhangi bir değer vermeden polinomal features a dönüştürmeliyiz
plt.plot(x,lin_reg2.predict(poly_reg.fit_transform(X)),color='blue')
plt.show()

#2 boyutlu uzayda x,y i dağıtma
plt.scatter(X,Y,color='red')
#x'in karşılığı olan lin_reg içinden predict etme
plt.plot(x,lin_reg.predict(X),color='blue')
plt.show()

plt.scatter(X,Y,color='red')
plt.plot(x,lin_reg3.predict(poly_reg3.fit_transform(X)),color='blue')
plt.show()

#predicts

print(lin_reg.predict(11))
print(lin_reg.predict(6.6))

print(lin_reg2.predict(poly_reg.fit_transform(11)))
print(lin_reg2.predict(poly_reg.fit_transform(6.6)))

#verilerin ölceklenmesi
from sklearn.preprocessing import StandardScaler
sc1=StandardScaler()
x_scaler=sc1.fit_transform(X)# x_train kumesini al buna buna standardscaler'ı uygula ve yeni uygulanmis halini X_train kumesi olarak yap
sc2=StandardScaler()
y_scaler=sc2.fit_transform(Y)

#SVR Regresyon
from sklearn.svm import SVR
svr_reg=SVR(kernel='rbf')
svr_reg.fit(x_scaler,y_scaler)

plt.scatter(x_scaler,y_scaler,color='red')
plt.plot(x_scaler,svr_reg.predict(x_scaler),color='blue')
plt.show()
print(svr_reg.predict(11))
print(svr_reg.predict(6.6))

#Decision Tree Regresyon
from sklearn.tree import DecisionTreeRegressor
r_dt=DecisionTreeRegressor(random_state=0)
r_dt.fit(X,Y)#X'ten Y'yi öğren
Z=X+0.5
K=X-0.4
W=X+0.1
plt.scatter(X,Y,color='red')
plt.plot(x,r_dt.predict(X),color='blue')
plt.plot(x,r_dt.predict(Z),color='green')
plt.plot(x,r_dt.predict(K),color='yellow')
plt.plot(x,r_dt.predict(W),color='black')
plt.show()
print(r_dt.predict(11))
print(r_dt.predict(6.6))

#Random Forest Regresyonu
from sklearn.ensemble import RandomForestRegressor

rf_reg=RandomForestRegressor( n_estimators=10,random_state=0)
rf_reg.fit(X,Y)
print(rf_reg.predict(6.5))

plt.scatter(X,Y,color='red')
plt.plot(x,rf_reg.predict(X),color='blue')
plt.plot(x,rf_reg.predict(Z),color='yellow')
plt.plot(x,rf_reg.predict(K),color='green')
plt.show()

print("Random Forest R2 Değeri")
print(r2_score(Y,rf_reg.predict(X))) #gerçek Y(nin alacağı değerler) değerler actual değerler
print(r2_score(Y,rf_reg.predict(K)))
print(r2_score(Y,rf_reg.predict(Z)))

#summary R2 değerleri
print("*************")
print("Linear Regression R2 Değeri")
print(r2_score(Y,lin_reg.predict(X)))

print("Polynomial Regression R2 Değeri")
print(r2_score(Y,lin_reg2.predict(poly_reg.fit_transform(X))))

print("SVR R2 Değeri")
print(r2_score(y_scaler,svr_reg.predict(x_scaler)))

print("Decision Tree R2 Değeri")
print(r2_score(Y,r_dt.predict(X)))

print("Random Forest R2 Değeri")
print(r2_score(Y,rf_reg.predict(X)))


# coding: utf-8

"""
Perform inference with a CD UNet/UNet++ model for for Sentinel-2 datasets

@Author: Tony Di Pilato

Created on Wed Feb 19, 2020
"""

import os
import numpy as np
from tensorflow import keras as K
import cdUtils
import cdModels
from osgeo import gdal
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--size', type=int, default=128)
parser.add_argument('-cpt', type=int, default=500) # Number of crops per tiff
parser.add_argument('--channels', '-ch', type=int, default=13)
parser.add_argument('--loss', '-l', type=str, default='bce', help='bce, wbced or dice')
parser.add_argument('--model', type=str, default='EF', help='EF, Siam or SiamDiff')
parser.add_argument('--city', type=str, default='rome', help='rome or sot')
parser.add_argument('--year', '-y', type=int, default='2015', help='from 2015 to 2019, take the pair thisyear-nextyear')
parser.add_argument('--histmatched', '-hm', type=bool, default='False', help='select True to use histmatched post images')

args = parser.parse_args()

img_size = args.size
channels = args.channels
cpt = args.cpt
mod = args.model
loss = args.loss
city = args.city
year = str(args.year)+'-'+str(args.year + 1)
hm = args.histmatched
classes = 1
model_dir = '../models/' + mod + '/'

model_name = mod+'_'+str(img_size)+'_cpt-'+str(cpt)+'-'+loss+'_'+str(channels)+'channels'
model_dir = model_dir + model_name + '/'
model_name = model_name + '-final'

# Load the model
if(loss=='wbced'):
    model = K.models.load_model(model_dir + model_name + '.h5', custom_objects={'weighted_bce_dice_loss': cdModels.weighted_bce_dice_loss})
else:
    model = K.models.load_model(model_dir + model_name + '.h5')
model.summary()
print("Model loaded!")

if(city=='rome'):
    dataset_dir = '../../CD_Sentinel2/tiff/Rome/'
    infres_dir = '../results/rome/'
    cm_name = 'Rome-' + model_name
else:
    if(hm):
        dataset_dir = '../Stoke_On_Trent/histmatched/' + year + '/'
        cm_name = 'SoThm-' + model_name + '_' + year
    else:
        dataset_dir = '../Stoke_On_Trent/' + year + '/'
        cm_name = 'SoT-' + model_name + '_' + year
    infres_dir = '../results/SoT/'

os.makedirs(infres_dir, exist_ok=True)

for image in os.listdir(dataset_dir):
    if((image.startswith('2') or image.startswith('p')) and ('pre' in image)):
        img_pre = image
    elif((image.startswith('2') or image.startswith('p')) and ('post' in image)):
        img_post = image

print(img_pre)
print(img_post)

tiffData_1 = gdal.Open(dataset_dir + img_pre)
raster1 = cdUtils.build_raster_fromMultispectral(tiffData_1, channels)
tiffData_2 = gdal.Open(dataset_dir + img_post)
raster2 = cdUtils.build_raster_fromMultispectral(tiffData_2, channels)
raster = np.concatenate((raster1,raster2), axis=2)
padded_raster = cdUtils.pad(raster, img_size)
test_image = cdUtils.crop(padded_raster, img_size, img_size)

# Create inputs for the Neural Network
inputs = np.asarray(test_image, dtype='float32')
inputs_1 = inputs[:,:,:,:channels]
inputs_2 = inputs[:,:,:,channels:]
inputs = [inputs_1, inputs_2]

# Perform inference
results = model.predict(inputs)

# Build the complete change map
shape = (padded_raster.shape[0], padded_raster.shape[1], classes)
padded_cm = cdUtils.uncrop(shape, results, img_size, img_size)
cm = cdUtils.unpad(raster.shape, padded_cm)

cm = np.squeeze(cm)
cm = np.rint(cm) # we are only interested in change/unchange
cm = cm.astype(np.uint8)

# Now create the georeferenced change map
cdUtils.createGeoCM(infres_dir + cm_name + '.tif', tiffData_1, cm)

print('Georeferenced change map created at %s' %infres_dir)


# coding: utf-8
import os.path

from github import Github

from DataModule import *
from SmartThreshold import *

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)  # Log等级总开关
rq = time.strftime('%Y%m%d%H%M', time.localtime(time.time()))
log_path = os.path.dirname(os.path.realpath(__file__)) + '/log/'
log_name = log_path + rq + '.log'
logfile = log_name
fh = logging.FileHandler(logfile, mode='w')
fh.setLevel(logging.ERROR)  # 输出到file的log等级的开关

ch = logging.StreamHandler()
ch.setLevel(logging.INFO)  # 输出到console的log等级的开关
# 第三步，定义handler的输出格式
# formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
formatter = logging.Formatter("%(message)s")
fh.setFormatter(formatter)
ch.setFormatter(formatter)

# 第四步，将logger添加到handler里面
logger.addHandler(ch)
logger.addHandler(fh)
# 日志
logger.debug('this is a logger debug message')

# 思路如下：
# 1.根据stars和folk的数量进行筛选
# 2.
# 需要每次从文本中读出来

cf = configparser.ConfigParser()
cf.read("./conf/application.conf")

auth = cf.get('github', 'github_token')

github = Github(auth)

RepositoriesModel.create_table()
OwnerModel.create_table()
ContributorModel.create_table()
OrgModel.create_table()
Repo_Owner_Rel.create_table()
Repo_Org_Rel.create_table()
Repo_Contributor_Rel.create_table()
Owner_Org_Rel.create_table()
Contributor_Org_Rel.create_table()
LastQueryConfig.create_table()

lastQueryConfig = None
print(auth)

def printRepository(repository):
    print("name:", repository.name)
    print("start count:", repository.stargazers_count)
    print("subscribers count:", repository.subscribers_count)
    print("watchers count:", repository.watchers_count)
    print("language:", repository.language)
    print("id:", repository.id)
    print("full name:", repository.full_name)
    print("forks count:", repository.forks_count)
    print("created at:", repository.created_at)
    print("description", repository.description)
    print("=========================================================")

@SmartThreshold.count_keep_rate(github)
def processOwnerOrgs(owner):
    return owner.get_orgs()

@SmartThreshold.count_keep_rate(github)
def processContributors(repo):
    return repo.get_contributors()

@SmartThreshold.count_keep_rate(github)
def processContributorOrgs(contributor):
    return contributor.get_orgs()

@SmartThreshold.count_keep_rate(github)
def processQuery(filterStr, condition, sort):
    repositories = github.search_repositories(filterStr, condition, "desc")
    for r in repositories:
        processRepo(r)

@SmartThreshold.count_keep_rate(github)
def processRepo(repo):
    global github
    printRepository(repo)
    mysql_db.connection()
    # with mysql_db.atomic():
    # 插入repo
    try:
        # 插入repo
        rm = RepositoriesModel().add_repo(repo)
        # 插入owner
        owner = OwnerModel().add_owner(repo.owner)
        # 增加repo和owner的关系
        Repo_Owner_Rel().add_repoownerrel(Repo_Owner_Rel(RepoID=repo.id, OwnerID=repo.owner.id))

        ownerorgs = processOwnerOrgs(repo.owner)
        if ownerorgs is not None:
            for oo in ownerorgs:
                try:
                    o = OrgModel().add_org(oo)
                    Owner_Org_Rel().add_owner_org_rel(Owner_Org_Rel(OrgID=oo.id, OwnerID=repo.owner.id))
                except Exception as e:
                    logger.error(e)

        contributors = processContributors(repo)
        if contributors is not None:
            for c in contributors[:10]:
                try:
                    # 增加贡献者
                    contributor = ContributorModel().add_contributor(c)
                    # 插入贡献者和repo之间的关系
                    Repo_Contributor_Rel().add_repo_contributor_rel(
                        Repo_Contributor_Rel(RepoID=repo.id, ContributorID=c.id))
                    con_orgs = processContributorOrgs(c)
                    if con_orgs is not None:
                        for org in con_orgs:
                            try:
                                o = OrgModel().add_org(org)
                                # 增加贡献者和org之间的关系
                                # 增加项目和org之间的关系
                                Contributor_Org_Rel().add_contributor_org_rel(
                                    Contributor_Org_Rel(OrgID=o.id, ContributorID=c.id))

                                Repo_Org_Rel().add_repo_org_rel(Repo_Org_Rel(RepoID=repo.id, OrgID=o.id))
                            except Exception as oe:
                                logger.error("error:{}".format(oe))
                except Exception as ce:
                    logger.error("error:{}".format(ce))
    except Exception as re:
        logger.error("error :{}".format(re))
    finally:
        mysql_db.commit()
        mysql_db.close()

# 根据一定的条件获取项目分布
# language:java|php|python|JavaScript
# condition:stars|forks|size
# step: per step,must >1
# min: the min range,must >1
# max: step*max = max range,must > 1
# displayAbove:is display the info more than max
def getRepoRangeFreq(language, condition, steps, min, max, displayAbove):
    global github

    if (displayAbove):
        filterStr = "{}:>={} language:{}".format(condition, (max + 1) * steps, language)
    for i in range(min, max + 1):
        filterStr = "{}:{}..{} language:{}".format(condition, (i - 1) * steps - 1, i * steps, language)
        logger.info(filterStr)
        LastQueryConfig().add_config(LastQueryConfig(startIdx=i, endIdx=max, steps=steps))
        try:
            processQuery(filterStr, condition, "desc")
        except Exception as e:
            time.sleep(60)
            logger.error(e)

# 根据fork获取项目分布
def getRepoForkRangeFreq(language):
    steps = 10
    min = 10
    max = 6000
    cfg = LastQueryConfig().get_last_config()
    if cfg is not None:
        min = cfg.startIdx
        max = cfg.endIdx
        steps = cfg.steps
    logger.info("start: steps:{},")
    getRepoRangeFreq(language, "stars", steps, min, max, False)

if __name__ == '__main__':
    getRepoForkRangeFreq("Java")


from typing import List
import math

from config import ModelParameters

class Utilities:
    """This class contains utility functions that are used in the project."""

    @staticmethod
    def get_output_sizes(
        res: int = ModelParameters.RESOLUTION,
        conv_kernel_size: int = ModelParameters.CONV_KERNEL_SIZE,
        conv_padding_size: int = ModelParameters.CONV_PADDING_SIZE,
        conv_stride: int = ModelParameters.CONV_STRIDE,
        mp_kernel_size: int = ModelParameters.MP_KERNEL_SIZE,
        mp_padding: int = ModelParameters.MP_PADDING_SIZE,
        mp_stride_length: int = ModelParameters.MP_STRIDE_LENGTH,
    ) -> List[int]:
        """
        This method is used to get the dimension of the output of each layer in the CNN. Configured to match the CNN architecture of the repo.

        Args:
            res (int, optional): input image resolution. Defaults to ModelParameters.RESOLUTION.
            conv_kernel_size (int, optional): Kernel size of the convolutional layer. Defaults to ModelParameters.CONV_KERNEL_SIZE.
            conv_padding_size (int, optional): Padding size of the convolutional layer. Defaults to ModelParameters.CONV_PADDING_SIZE.
            conv_stride (int, optional): Stride length of the convolutional layer. Defaults to ModelParameters.CONV_STRIDE.
            mp_kernel_size (int, optional): Kernel size of the Max Pooling layer. Defaults to ModelParameters.MP_KERNEL_SIZE.
            mp_padding (int, optional): Padding size of the Max Pooling layer. Defaults to ModelParameters.MP_PADDING_SIZE.
            mp_stride_length (int, optional): Stride length of the Max Pooling layer. Defaults to ModelParameters.MP_STRIDE_LENGTH.

        Returns:
            List[int]: List of the output dimensions for each layer in the CNN that are needed.
        """

        def get_layer_output_size(
            input_size: int, filter_size: int, padding: int, stride: int
        ) -> int:
            """
            Helper function to calculate the output size of a layer.

            Returns:
                int: Output size of the layer.
            """

            formula: int = math.floor(
                (((input_size - filter_size + 2 * padding) / stride) + 1)
            )

            return formula

        # Output size from applying conv1 to input
        conv1: int = get_layer_output_size(
            res, conv_kernel_size, conv_padding_size, conv_stride
        )

        # Output size from applying max pooling 1 to conv1
        mp1: int = get_layer_output_size(
            conv1, mp_kernel_size, mp_padding, mp_stride_length
        )

        # Output size from applying conv2 to max pooling 1
        conv2: int = get_layer_output_size(
            mp1, conv_kernel_size, conv_padding_size, conv_stride
        )

        # Output size from applying conv3 to conv 2
        conv3: int = get_layer_output_size(
            conv2, conv_kernel_size, conv_padding_size, conv_stride
        )

        # Output size from applying max pooling 2 to conv3
        mp2: int = get_layer_output_size(
            conv3, mp_kernel_size, mp_padding, mp_stride_length
        )

        # Output size from applying conv 4 to max pooling 2
        conv4: int = get_layer_output_size(
            mp2, conv_kernel_size, conv_padding_size, conv_stride
        )

        # Output size from applying conv5 to conv 4
        conv5: int = get_layer_output_size(
            conv4, conv_kernel_size, conv_padding_size, conv_stride
        )

        # Output size from applying max pooling 3 to conv 5
        mp3: int = get_layer_output_size(
            conv5, mp_kernel_size, mp_padding, mp_stride_length
        )

        # List of dimensions of outputs from each layer that I need.
        outputs_i_need: List[int] = [mp1, conv2, mp2, conv4, mp3]

        return outputs_i_need


import torch
import argparse
from torch.utils.data import DataLoader
from data_utils import ChoiceDataset
from models import TasteNetChoice
from train import train
from evaluation import summarize, printSummary, predictTastes, RMSE, ABSE, RE, RMSE_vector, ABSE_vector, RE_vector
from simulate import error_of_vot, error_of_vowt, dic_z, dic_z_z01, dic_z_zall, inc
import pickle
import os
import copy
import numpy as np
from regress_coef import regress
import itertools
from joblib import Parallel, delayed, parallel_backend

def one_run(data_train, data_dev, args, params):
    """
    A single model training run with one set of hyper-parameters and random initialization seed
    """
    # set up parameters
    # torch.manual_seed(params['seed'])
    input_size = 3  # z has 3 dim
    output_size = 1 if params['separate'] else 2
    args.layer_sizes = [input_size, params['hidden_size'], output_size]
    args.activation = params['activation']
    args.transform = params['transform']
    args.weight_decay = params['weight_decay']
    args.separate = params['separate']
    print(args)

    # train
    success = False
    while not success:
        # initialize model
        model = TasteNetChoice(args)
        _, _, model, success = train(model, data_train, data_dev, args, save=False)

    # evaluate
    result = one_run_summary(model, ds_train, ds_dev, ds_test)

    return result, model

def one_run_summary(model, ds_train, ds_dev, ds_test):
    # Metrics
    result = {}
    # 1. NLL and ACC for train/dev/test
    summary = summarize(model, ds_train, ds_dev, ds_test)
    result.update(summary)
    x = printSummary(summary, precision=3)
    print(x)

    # 2. Tastes (vots, vowts): predicted vs true (dollar per minute)
    allds = {"train": ds_train, "dev": ds_dev, "test": ds_test}

    pred_vots = [predictTastes(model, allds[name].z)['vots'] for name in ["train", "dev", "test"]]
    true_vots = [allds[name].vots for name in ["train", "dev", "test"]]

    result['vot_rmse'] = RMSE(pred_vots, true_vots).item()
    result['vot_mabse'] = ABSE(pred_vots, true_vots).item()
    result['vot_re'] = RE(pred_vots, true_vots).item()

    pred_vowts = [predictTastes(model, allds[name].z)['vowts'] for name in ["train", "dev", "test"]]
    true_vowts = [allds[name].vowts for name in ["train", "dev", "test"]]

    result['vowt_rmse'] = RMSE(pred_vowts, true_vowts).item()
    result['vowt_mabse'] = ABSE(pred_vowts, true_vowts).item()
    result['vowt_re'] = RE(pred_vowts, true_vowts).item()

    # 3. Tastes (vots, vowts) on simulated z: unit is dollar per hour!
    input_z = copy.deepcopy(dic_z)
    sim_pred_vots, sim_true_vots, rmse, mabse, re = error_of_vot(model, dic_z, input_z, ds_train.params)
    result['sim_vot_rmse'] = rmse
    result['sim_vot_mabse'] = mabse
    result['sim_vot_re'] = re

    sim_pred_vowts, sim_true_vowts, rmse, mabse, re = error_of_vowt(model, dic_z, input_z, ds_train.params)
    result['sim_vowt_rmse'] = rmse
    result['sim_vowt_mabse'] = mabse
    result['sim_vowt_re'] = re

    # 4. Coefficients from regression
    coefs_asc1 = model.getParams()[-1][0].item()
    coefs_time, coefs_wait = regress(model, dic_z, dic_z_zall)
    coefs_pred = [coefs_asc1] + coefs_time + coefs_wait

    coefs_asc1_true = ds_train.asc1
    coefs_time_true = ds_train.coefs_time
    coefs_wait_true = ds_train.coefs_wait
    coefs_true = [coefs_asc1_true] + coefs_time_true + coefs_wait_true

    coefs_pred = torch.Tensor(coefs_pred)
    coefs_true = torch.Tensor(coefs_true)

    rmse = RMSE_vector(coefs_pred, coefs_true)
    mabse = ABSE_vector(coefs_pred, coefs_true)
    re = RE_vector(coefs_pred, coefs_true)

    result['coef_rmse'] = rmse
    result['coef_mabse'] = mabse
    result['coef_re'] = re
    result['coef'] = coefs_pred
    result['coef_true'] = coefs_true
    return result

parser = argparse.ArgumentParser(description='TasteNet-MNL (A Toy Example)')
parser.add_argument("--data_dir", type=str, default="../toy_data")
parser.add_argument("--data_file", type=str, default="data_10k.pkl", help="data dictionary in pickle format")
parser.add_argument("--output_dir", type=str, default="../result")
parser.add_argument("--scenario", type=str)

parser.add_argument('--cuda', action='store_true', default=False,
                    help='enables CUDA training')

# Training params
parser.add_argument('--batch_size', type=int, default=64,
                    help='input batch size for training (default: 64)')
parser.add_argument('--num_epochs', type=int, default=100,
                    help='maximum number of epochs to train (default: 100)')
parser.add_argument("--lr", type=float, default=0.001, help="learning rate")
parser.add_argument("--nll_tol", type=float, default=0.0001, help="tolerance for nll convergence")

args = parser.parse_args()

cuda = args.cuda and torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")

# load data
data = pickle.load(open(os.path.join(args.data_dir, args.data_file), "rb"))

ds_train = ChoiceDataset(data['train'])
ds_dev = ChoiceDataset(data['dev'])
ds_test = ChoiceDataset(data['test'])

data_train = DataLoader(ds_train, batch_size=args.batch_size, shuffle=True, num_workers=5)
data_dev = DataLoader(ds_dev, batch_size=args.batch_size, shuffle=False, num_workers=5)
data_test = DataLoader(ds_test, batch_size=args.batch_size, shuffle=False, num_workers=5)

# Set hyperparam_candidates
hyperparam_candidates = dict(
        hidden_size=[200],
        weight_decay=[0.0001, 0.001],
        activation=["relu"],
        transform=["relu"],
        separate=[False],
        no=range(100) # number of runs with different random initialization
    )

hyperparam_sets = [dict(zip(hyperparam_candidates.keys(), values))
                       for values
                       in itertools.product(*hyperparam_candidates.values())]
print(f"{len(hyperparam_sets)} set of params")
print(hyperparam_sets)

result_all = []
model_all = []
for params in hyperparam_sets:
    result, model = one_run(data_train, data_dev, args, params)
    result_all.append(result)
    model_all.append(model)

evals_name = f"{args.scenario}_evals.pkl"
models_name = f"{args.scenario}_models.pkl"
args_name = f"{args.scenario}_args.pkl"
hyper_name = f"{args.scenario}_hyper.pkl"

pickle.dump(result_all, open(os.path.join(args.output_dir, evals_name), "wb"))
pickle.dump(model_all, open(os.path.join(args.output_dir, models_name), "wb"))
pickle.dump(hyperparam_sets, open(os.path.join(args.output_dir, hyper_name), "wb"))
pickle.dump(args, open(os.path.join(args.output_dir, args_name), "wb"))


from django.urls import include, path
from rest_framework.routers import SimpleRouter

from .views import AuthViewset, UsersViewset

app_name = 'api'

router_v1 = SimpleRouter()
router_v1.register('auth', AuthViewset, 'auth')
router_v1.register('users', UsersViewset, 'users')

urlpatterns = [
    path('', include(router_v1.urls))
]


import Preprocessing
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score

#Train model with KNN
X_train,X_test,y_train,y_test = Preprocessing.X_train,Preprocessing.X_test,Preprocessing.y_train,Preprocessing.y_test

#Find k in range value
def Knn(x,y,left,right):
    knn_score = []
    for i in range (left,right):
        knn = KNeighborsClassifier(n_neighbors = i,weights = 'uniform')
        knn.fit(x,y)
        y_hat = knn.predict(X_test)
        knn_score.append(accuracy_score(y_hat,y_test))
    return knn_score

# After run knn function, choose k = 65 for training
knn=KNeighborsClassifier(n_neighbors=65)
knn.fit(X_train,y_train)
y_hat = knn.predict(X_test)
# print('Accuracy on train set: {}'.format(accuracy_score(y_train,knn.predict(X_train))))
# print('Accuracy on test set: {}'.format(accuracy_score(y_hat,y_test)))

# Accuracy on train set: 0.7257531227038942
# Accuracy on test set: 0.7170758586909937


#!/usr/bin/env python3

# REQUIRES PYTHON >= 3.8

"""
This is a partial solution for the doors with the tiles, located near where
the bodies in barrels were found. This door leads to the tunnels below the
manor house.

The game is a rotational tile game. Tiles are grouped into sets of 4 around a
central button. When the button is pressed, the tiles in the group of 4 shift
counter-clockwise. Each button shares two of its tiles with the buttons below
it and above it, respectively (e.g., in an array of 6 tiles, button 1 controls
the top 2 and the middle 2; button 2 controls the middle 2 and bottom 2).

The standard board looks as follows, where each number represents a
symbol that has been etched on the board:

    c1                c6
          button 1
    c2                c7
          button 2
    c3                c8
          button 3
    c4                c9
          button 4
    c5                c10

For example, button 1 controls spaces 1, 2, 7, and 6 (listed counter-clockwise
starting with the top left). When combined with the tiles sitting in the slots
to the outside, the board looks something like:

    t8:c1              t7:c6
            button 1
    t2:c2              t3:c7
            button 2
    t10:c3             t4:c8
            button 3
    t5:c4              t9:c9
            button 4
    t6:c5              t1:c10

When button 1 is pushed, the board will look like this:

    t7:c1              t3:c6
            button 1
    t8:c2              t2:c7
            button 2
    t10:c3             t4:c8
            button 3
    t5:c4              t9:c9
            button 4
    t6:c5              t1:c10

If we then pushed button 2, the board will look like:

    t7:c1              t3:c6
            button 1
    t2:c2              t4:c7
            button 2
    t8:c3              t10:c8
            button 3
    t5:c4              t9:c9
            button 4
    t6:c5              t1:c10

The purpose of the game is to rotate the tiles until the symbols on the tiles
(represented by numbers here) match the control symbols next to their
respective slots:

    t1:c1              t6:c6
            button 1
    t2:c2              t7:c7
            button 2
    t3:c3              t8:c8
            button 3
    t4:c4              t9:c9
            button 4
    t5:c5              t10:c10

There are 10 tiles in the original game, with 4 buttons to rotate the groups
of tiles.
"""

from typing import Dict, Final, List, Tuple, TypedDict

class ControlsConfig(TypedDict):
    button: int
    controls: list[int]

BoardConfig = List[ControlsConfig]
TilePosition = Dict[int, int]
TileShifts = Dict[int, List[Tuple[int, int]]]
StepList = List[int]
StepQueue = List[Tuple[StepList, TilePosition]]

# This is the initial setup in Enigmatis 2
DEFAULT_BOARD_CONFIG: Final[BoardConfig] = [
    {
        "button": 1,
        "controls": [1, 2, 7, 6],
    },
    {
        "button": 2,
        "controls": [2, 3, 8, 7],
    },
    {
        "button": 3,
        "controls": [3, 4, 9, 8],
    },
    {
        "button": 4,
        "controls": [4, 5, 10, 9],
    },
]
DEFAULT_TILE_POSITION: Final[TilePosition] = {
    1: 10,
    2: 2,
    3: 9,
    4: 5,
    5: 3,
    6: 1,
    7: 6,
    8: 7,
    9: 4,
    10: 8,
}

class Solver:
    def __init__(self, config: BoardConfig=DEFAULT_BOARD_CONFIG,
                 starting_tile_position: TilePosition=DEFAULT_TILE_POSITION):
        """
        NOTE: this init does not check for correct starting positions and
        configurations. You should be using some configuration from Enigmatis
        2:

            1. An even number of tile positions, more than 4, but less than 10
            2. The tile positions should match the numbers in the config
               controls

        :param dict starting_tile_position: describes the starting position of
            the tiles relative to the control positions where they belong. For
            example, if you have a circle tile in control position 1, and a
            triangle tile in control position 2, but their positions should be
            reversed, your dictionary should look like:

                {
                    1: 2,  # control position: tile position
                    2: 1,
                }

        :param list config: describes the layout of the puzzle, showing which
            buttons map rotate which control positions. The STANDARD_CONFIG
            assumes numbering the control positions down the left side first
            (1-5), then down the right side (6-10). The buttons that control
            the positions are numbered top-to-bottom; the controls groups are
            listed in counter-clockwise order, starting from the top left.
        """
        self.config: Final[BoardConfig] = config

        # This will be set as part of run(), and is displayed by str() upon
        # completion
        self.solution: List[int] = []

        # Internal use during run()
        self._allowed_tile_shifts: TileShifts = self._generate_tile_shifts()
        self._current_layout: TilePosition = None
        self._step_queue: StepQueue = [([], starting_tile_position)]
        self._seen_layouts: Set[int] = set()

    def __str__(self) -> str:
        tostring = '\n----------\n\n'

        if not self.solution:
            tostring += 'The puzzle is not solved.\n\n'
            tostring += f'Current layout:\n\t{self._current_layout}'
        else:
            tostring += 'The steps to solve the puzzle:\n\n'
            for step_num, step in enumerate(self.solution):
                tostring += '\t{}. Press button {}\n'.format(
                    self._format_integer(step_num + 1),
                    step,
                )

        tostring += '\n\n---------\n'
        return tostring

    def run(self):
        while not self.solution and len(self._step_queue) > 0:
            current_steps, current_tile_position = self._step_queue.pop()
            self._current_layout = current_tile_position

            next_steps = self._generate_next_steps(
                current_steps,
                current_tile_position,
            )
            for steps, tile_position in next_steps:
                if self._is_solved(tile_position):
                    self.solution = steps
                    self._current_layout = tile_position
                    break

                if self._is_improvement(tile_position):
                    self._step_queue.insert(0, [steps, tile_position])
                else:
                    # We don't want to waste our time in the future, so let's
                    # stick it in seen_layouts, even though we haven't properly
                    # looked at it
                    self._seen_layouts.add(
                        self._format_seen_layout(tile_position),
                    )

            self._seen_layouts.add(
                self._format_seen_layout(current_tile_position),
            )

        print(self)

    def _calculate_new_position(self, tile_position: TilePosition,
                                button_group: ControlsConfig) -> TilePosition:
        """
        When a button is pressed, calculate the new positions for tiles
        """
        new_position = tile_position.copy()
        shifts = self._allowed_tile_shifts[button_group['button']]

        for old_tile, new_tile in shifts:
            new_position[old_tile] = tile_position[new_tile]

        return new_position

    def _format_integer(self, num: int) -> str:
        """
        Pad out integers less than 10 with a leading 0
        """
        return str(num).zfill(2)

    def _format_seen_layout(self, tile_position: TilePosition) -> int:
        """
        Converts a dictionary into an integer for self.seen_layouts.

        This is to minimize comparison times when checking "Have we seen this
        layout before?"
        """
        pos_list = list(tile_position.items())
        pos_list.sort()
        return int( '1' + ''.join([
            self._format_integer(num)
            for tup in pos_list
                for num in tup
        ]))

    def _generate_next_steps(self, step_list: StepList,
                             tile_position: TilePosition) -> StepQueue:
        new_queue = []

        for button_group in self.config:
            new_steps = step_list[:] + [button_group['button']]
            new_position = self._calculate_new_position(
                tile_position,
                button_group,
            )

            new_queue.append((new_steps, new_position))

        return new_queue

    def _generate_tile_shifts(self) -> TileShifts:
        """
        Used by init to generate the ways tiles can shift at button press
        """
        shifts_dict = {}

        for control_config in self.config:
            button = control_config['button']
            old_position = control_config['controls']

            new_position = old_position[:]
            new_position.insert(0, new_position.pop())
            shifts_dict[button] = list(zip(old_position, new_position))

        return shifts_dict

    def _is_improvement(self, tile_position: TilePosition) -> bool:
        if self._format_seen_layout(tile_position) in self._seen_layouts:
            return False

        # TODO: I was originally going to play around with a genetic algorithm
        # that calculated the distance to the tile's final location, and then
        # prune any options that were a significant decrease in score (i.e., it
        # gets farther from the desired final outcome; however, the Solver
        # already returns a result relatively quickly, and a genetic algorithm
        # is a bit of an overkill if this solver isn't taking a long time.
        # Perhaps I'll play with genetic algorithms later?
        return True

    def _is_solved(self, tile_position: TilePosition) -> bool:
        for control, tile in tile_position.items():
            if control != tile:
                return False
        return True

if __name__ == '__main__':
    # Don't run this unless the appropriate version of python is being used
    import sys
    if not (sys.version_info.major == 3 and sys.version_info.minor >= 8):
        print("You must have at least python version 3.8 to run this.")
        sys.exit(1)

    Solver().run()


"""This module will serve the api request."""

from config import client, ObjectId
from app import app
from bson.json_util import dumps
from flask import request, jsonify, send_file, make_response
import json
import ast
import imp
import time
import codecs

# Import the helpers module
helper_module = imp.load_source('*', './app/helpers.py')

# Select the database
db = client.incidents
# Select the collection
collection = db.reported_incidents

INCIDENT_TO_CODE = {'tornado' : 1,
					'flood' : 2,
					'wildfire' : 3,
					'earthquake' : 4,
					'tsunami' : 5,
					'storm' : 6,
					'drought' : 7,
					'avalanche' : 8,
					'landslide' : 9,
					'sinkhole' : 10,
					'volcanic_eruption' : 11,
					'thunderstorm' : 12}
CODE_TO_INCIDENT = ['', 'tornado', 'flood', 'wildfire', 'earthquake', 'tsunami',
				 'storm', 'drought', 'avalanche', 'landslide', 'sinkhole',
				 'volcanic_eruption', 'thunderstorm']
GET_ALL_SUMMARY = {'latitude' : 1, 'longitude' : 1, 'type' : 1}
IMAGE_FOLDER = '/home/cristi/Desktop/Proiect_BD/images/'
IMAGE_EXTENSION = '.jpeg'

@app.route("/")
def get_initial_response():
    """Welcome message for the API."""
    # Message to the user
    message = {
        'apiVersion': 'v1.0',
        'status': '200',
        'message': 'Welcome to the Incidents Reporting Service!'
    }
    # Making the message looks good
    resp = jsonify(message)
    # Returning the object
    return resp

'''
Demo body:
{"type": "tornado","description": "Second incident","latitude": 44.467645,"longitude": 26.140665,"tags": ["wow","incident","again"]}
'''
@app.route("/api/v1/incidents", methods=['POST'])
def add_incident():
    try:
        # Create new incidents
        try:
            incident_str = request.form['data']
            image = request.files['image']
            incident = ast.literal_eval(incident_str)
        except:
            # Bad request as request body is not available
            return "", 400

        #print incident
        # Convert natural disaster name to corresponding code
        incident['code'] = INCIDENT_TO_CODE[incident['type']]

        # Add the image path to the incident information
        incident['image_path'] = ''

        # Add timestamp
        incident['timestamp'] = time.time()

        # Insert incident into mongo and get its ID
        record_id = collection.insert(incident)
        if record_id is None:
            raise IOError("Could not insert document into MongoDB")
        record_id = str(record_id)

        # Save the image to disk
        image_name = record_id + IMAGE_EXTENSION
        res = helper_module.save_filestorage_object(image, IMAGE_FOLDER, image_name)
        if not res:
            # Could not save the incident image. Roll back mongo
            collection.delete_one({"_id" : ObjectId(record_id)})
            raise IOError("Could not save file on disk")

        # Update the image path from Mongo Document
        image_path = IMAGE_FOLDER + image_name
        record = collection.find_one_and_update({"_id" : ObjectId(record_id)},
            {"$set" : {"image_path" : image_path}})

        # Return ID of the newly created item
        return jsonify(record_id), 201

    except:
        # Error while trying to create the resource
        return "", 500

@app.route("/api/v1/incidents", methods=['GET'])
def get_incidents():
    try:
        # Call the function to get the query params
        query_params = helper_module.parse_query_params(request.query_string)
        #print query_params
        # Check if dictionary is not empty
        if query_params:

            # Get all information about specific incident
            query = {"_id" : ObjectId(query_params["_id"])}
            #print query
            incident = collection.find_one(query)

            # Check if the records are found
            if incident:
                # Prepare the response
                #return dumps(incident)
                #image_data = codecs.open(incident['image_path'], 'rb').read()
                #data = {}
                #data['data'] = incident
                #data['image'] = image_data
                #resp = make_response()
                #resp.files = image_data
                #resp.data = incident
                #resp.data = data
                #return resp
                #return send_file(incident['image_path'])#, resp
                response = send_file(incident['image_path'])
                #response.form = incident
                response.headers['incident_data'] = str(incident)
                #print (len(response.data))
                return response
            else:
                # No records are found
                return "", 404

        # If dictionary is empty
        else:
            # Return all the records as query string parameters are not available
            if collection.find().count > 0:
                # Prepare response if the incidents are found
                return dumps(collection.find({}, GET_ALL_SUMMARY))
            else:
                # Return empty array if no incidents are found
                return jsonify([])
    except Exception as e:
        # Error while trying to fetch the resource
        return str(e), 500

@app.route("/api/v1/incidents/<incident_id>", methods=['DELETE'])
def remove_incident(incident_id):
    try:
        # Delete the incident
        delete_inc = collection.delete_one({"_id" : ObjectId(incident_id)})
        image_path = IMAGE_FOLDER + incident_id + IMAGE_EXTENSION
        helper_module.remove_file(image_path)

        if delete_inc.deleted_count > 0 :
            # Prepare the response
            return "", 204
        else:
            # Resource Not found
            return "", 404
    except:
        # Error while trying to delete the resource
        return "", 500

@app.route("/api/v1/users/<user_id>", methods=['POST'])
def update_user(user_id):
    """
       Function to update the user.
       """
    try:
        # Get the value which needs to be updated
        try:
            body = ast.literal_eval(json.dumps(request.get_json()))
        except:
            # Bad request as the request body is not available
            # Add message for debugging purpose
            return "", 400

        # Updating the user
        records_updated = collection.update_one({"id": int(user_id)}, body)

        # Check if resource is updated
        if records_updated.modified_count > 0:
            # Prepare the response as resource is updated successfully
            return "", 200
        else:
            # Bad request as the resource is not available to update
            # Add message for debugging purpose
            return "", 404
    except:
        # Error while trying to update the resource
        # Add message for debugging purpose
        return "", 500

@app.errorhandler(404)
def page_not_found(e):
    """Send message to the user with notFound 404 status."""
    # Message to the user
    message = {
        "err":
            {
                "msg": "This route is currently not supported. Please refer API documentation."
            }
    }
    # Making the message looks good
    resp = jsonify(message)
    # Sending OK response
    resp.status_code = 404
    # Returning the object
    return resp


import torch
import torch.nn as nn
import torch.nn.functional as F

import numpy as np
from scipy.stats import truncnorm

def get_truncated_noise(n_samples, z_dim, truncation):
    """Function for creating truncated noise vectors: Given dimensions (n_samples, z_dim)
        and truncation value, cretes a tensor of that shape filled with random
        numbers from the truncated normal distribution.

    Args:
        n_samples (int): the number of samples to generate
        z_dim (int): the dimension of the noise vector
        truncation (float): the truncation value
    """
    truncated_noise = truncnorm.rvs(-truncation, truncation, size=(n_samples, z_dim))
    return torch.Tensor(truncated_noise)

class MappingLayers(nn.Module):
    """Mapping Layers Class

    Args:
        z_dim (int): the deimension of the noise vector
        hidden_dim (int): the inner dimension
        w_dim (int): the dimension of the intermediate noise vector (w)
    """
    def __init__(self, z_dim, hidden_dim, w_dim):
        super().__init__()
        self.mapping = nn.Sequential(
            nn.Linear(z_dim, hidden_dim),
            nn.ReLU(inplace=True),
            nn.Linear(hidden_dim, hidden_dim),
            nn.ReLU(inplace=True),
            nn.Linear(hidden_dim, w_dim),
        )

    def forward(self, noise):
        """Method for completing a forward pass of MappingLayers:
            Given an inital noise tensor, returns the intermediate noise tensor.

        Args:
            noise (tensor): A noise tensor with dimensions (n_samples, z_dim)
        """
        return self.mapping(noise)

class InjectNoise(nn.Module):
    """Inject Noise Class

    Args:
        channels (int): the number of channels the image has
    """
    def __init__(self, channels):
        super().__init__()
        weight = torch.empty(channels).normal_(mean=0, std=1)
        self.weight = nn.Parameter(torch.reshape(weight, (1, channels, 1, 1)))

    def forward(self, image):
        """Function for completing a forward pass of InjectNoise: Given an image,
            returns the image with random noise added.

        Args:
            image (tensor): the feature map of shape (n_samples, channels, width, height)
        """
        noise_shape = (image.shape[0], 1, image.shape[2], image.shape[3])
        noise = torch.randn(noise_shape, device=image.device)
        return image + (self.weight * noise)

class AdaIN(nn.Module):
    """AdaIn Class

    Args:
        channels (int): the number of channels the image has
        w_dim (int): the dimension of the intermediate noise vector
    """
    def __init__(self, channels, w_dim):
        super().__init__()
        self.instance_norm = nn.InstanceNorm2d(channels)
        self.style_scale_transform = nn.Linear(w_dim, channels)
        self.style_shift_transform = nn.Linear(w_dim, channels)

    def forward(self, image, w):
        """Method for completing a forward pass of AdaIN: Given an image and the intermediate noise vector w,
            returns the normalized image that has been scaled an shifted by the style

        Args:
            image (tensor): the feature map of shape (n_samples, channels, width, height)
            w (tensor): the intermediate noise vector
        """
        normalized_image = self.instance_norm(image)
        style_scale = self.style_scale_transform(w)[:, :, None, None]
        style_shift = self.style_shift_transform(w)[:, :, None, None]
        transformed_image = (normalized_image * style_scale) + style_shift

        return transformed_image

class MicroStyleGANGeneratorBlock(nn.Module):
    """Micro StyleGAN Generator Block Class

    Args:
        in_chan (int): the number of channels in the input
        out_chan (int): the number of channels wanted in the output
        w_dim (int): the dimension of the intermediate noise vector
        kernel_size (int): the size of the convolutional kernel
        starting_size (int): the size of the starting image
    """
    def __init__(self, in_chan, out_chan, w_dim, kernel_size, starting_size, use_upsample=True):
        super().__init__()
        self.use_upsample = use_upsample

        if self.use_upsample:
            self.upsample = nn.Upsample((starting_size, starting_size), mode="bilinear")
        self.conv = nn.Conv2d(in_chan, out_chan, kernel_size, padding=1)
        self.inject_noise = InjectNoise(out_chan)
        self.adain = AdaIN(out_chan, w_dim)
        self.activation = nn.LeakyReLU(negative_slope=0.2, inplace=True)

    def forward(self, x, w):
        """Function for completing a forward pass of MicroStyleGANGeneratorBlock: Given an x, and w,
        computes a StyleGAN generator block

        Args:
            x (tensor): the input into the generator, a feature map of sjhape (n_samples, channels, width, height)
            w (tensor): the intermediate noise vector
        """

        if self.use_upsample:
            x = self.upsample(x)
        x = self.conv(x)
        x = self.inject_noise(x)
        x = self.activation(x)
        x = self.adain(x, w)
        return x

class MicroStyleGANGenerator(nn.Module):
    """Micro StyleGAN Generator Class

    Args:
        z_dim (int): the dimension of the noise vector
        map_hidden_dim (int): the mapping inner dimension
        w_dim (int): the dimension of the intermediate noise vector
        in_chan (int): the number of channels in the input
        out_chan (int): the number of channels wanted in the output
        kernel_size (int): the size of the convolutional kernel
        hidden_chan (int): the inner dimension
    """
    def __init__(self, z_dim, map_hidden_dim, w_dim, in_chan, out_chan, kernel_size, hidden_chan):
        super().__init__()
        self.map = MappingLayers(z_dim, map_hidden_dim, w_dim)
        self.starting_constant = nn.Parameter(torch.randn(1, in_chan, 4, 4))
        self.block0 = MicroStyleGANGeneratorBlock(in_chan, hidden_chan, w_dim, kernel_size, 4, use_upsample=False)
        self.block1 = MicroStyleGANGeneratorBlock(hidden_chan, hidden_chan, w_dim, kernel_size, 8)
        self.block2 = MicroStyleGANGeneratorBlock(hidden_chan, hidden_chan, w_dim, kernel_size, 16)

        self.block1_to_image = nn.Conv2d(hidden_chan, out_chan, kernel_size=1)
        self.block2_to_image = nn.Conv2d(hidden_chan, out_chan, kernel_size=1)
        self.alpha = 0.2

    def upsample_to_match_size(self, smaller_image, bigger_image):
        """Function for upsampling an image to the size of another: Given two images (smaller and bigger),
            upsamples the first to have the same dims as the second one.

        Args:
            smaller_image (tensor): the smaller image to upsample
            bigger_image (tensor): the bigger images, whose dimension will be upsampled to
        """
        return F.interpolate(smaller_image, size=bigger_image.shape[-2:], mode="bilinear")

    def forward(self, noise, return_intermediate=False):
        """Method for completing a forward pass of MicroStyleGANGenerator: Given noise,
            computes a StyleGan interation.

        Args:
            noise (tensor): Noise tensor with dimensions (n_samples, z_dim)
            return_intermediate (bool, optional): True to return images (for testing)
        """
        x = self.starting_constant
        w = self.map(noise)
        x = self.block0(x, w)
        x_small = self.block1(x, w)
        x_small_image = self.block1_to_image(x_small)
        x_big = self.block2(x_small, w)
        x_big_image = self.block2_to_image(x_big)
        x_small_upsample = self.upsample_to_match_size(x_small_image, x_big_image)

        interpolation = torch.lerp(x_small_upsample, x_big_image, self.alpha)

        if return_intermediate:
            return interpolation, x_small_upsample, x_big_image
        return interpolation


import requests
from bs4 import BeautifulSoup
import json
import re
import collections

HOST = "https://www.cfcunderwriting.com"

def get_privacy_policy_location(all_urls):
    for _, url in enumerate(all_urls):
        if "privacy-policy" in url:
            return url

def get_external_resources(all_urls):
    external_resources = []
    for url in all_urls:
        if is_external(url):
            external_resources.append(url)

    return external_resources

def is_external(url):
    return (url.startswith("http") or url.startswith("//")) and "cfcunderwriting" not in url

def get_url(tag, attr):
    return tag[attr]

def has_url(tag):
    return tag.has_attr("href") or tag.has_attr("src")

def write_to_json_file(filename, content):
    with open(filename, "w") as outfile:
        json.dump(content, outfile)
    outfile.close()

def scrape_page(url):
    r = requests.get(url)
    return BeautifulSoup(r.content, "html.parser")

def main():
    # scrape index page
    index_page = scrape_page(HOST)
    all_resources = index_page.find_all(has_url)  # full tags
    all_urls = [get_url(r, "href") if r.has_attr("href") else get_url(r, "src") for r in all_resources]  # urls

    # list of external resources
    external_resources = get_external_resources(all_urls)
    write_to_json_file("external_resources.json", external_resources)

    privacy_page_location = get_privacy_policy_location(all_urls)

    # scrape privacy policy page
    privacy_page = scrape_page(HOST + privacy_page_location)
    text = privacy_page.get_text(separator=" ", strip=True)
    # use regex to strip the text of special characters and numbers
    stripped_text = re.sub("[^a-zA-Z\s]", "", text)
    text_list = stripped_text.lower().split()
    # create a Counter object to calculate word frequencies
    frequencies = collections.Counter(text_list)
    write_to_json_file("word_count.json", dict(frequencies))

if __name__ == "__main__":
    main()


from maya import cmds
from ngSkinTools import selectionState
from ngSkinTools.doclink import SkinToolsDocs
from ngSkinTools.influenceMapping import InfluenceMapping
from ngSkinTools.log import getLogger
from ngSkinTools.mllInterface import MllInterface
from ngSkinTools.ui.basetab import CommandLayout
from ngSkinTools.ui.basetoolwindow import BaseToolWindow
from ngSkinTools.ui.components import uiGroup
from ngSkinTools.ui.components.influencesManualMapping import InfluencesManualMapping
from ngSkinTools.ui.components.influencesMappingPreview import InfluencesMappingPreview

log = getLogger('editMirrorInfluencesWindow')

class EditMirrorInfluencesWindow(BaseToolWindow):
    def __init__(self, windowName):
        BaseToolWindow.__init__(self, windowName)
        self.useUserPrefSize = False
        self.windowTitle = 'Edit Mirror Influences'
        self.sizeable = True
        self.defaultHeight = 400
        self.defaultWidth = 450

        self.influenceMappingPreview = InfluencesMappingPreview()
        self.influencesManualMapping = InfluencesManualMapping()

        self.buildMapper = None

        self.mapper = InfluenceMapping()

    @staticmethod
    def getInstance():
        return BaseToolWindow.getWindowInstance('EditMirrorInfluencesWindow', EditMirrorInfluencesWindow)

    def setMapperBuilder(self,buildMapper):
        self.buildMapper = buildMapper
        self.previewInfluenceMapping()

    def createWindow(self):
        BaseToolWindow.createWindow(self)

        buttons = [
            ('Close', self.closeWindow, '')
        ]

        self.cmdLayout = CommandLayout(SkinToolsDocs.MIRRORWEIGHTS_INTERFACE, buttons)

        contentRoot = self.cmdLayout.innerLayout

        self.influenceMappingPreview.mirrorMode = True
        self.influenceMappingPreview.createUI(contentRoot)
        self.influenceMappingPreview.onDelete.addHandler(self.influencesManualMapping.removeSelectedManualMappings, contentRoot)

        manualGroup = uiGroup.create(contentRoot, 'Manual influence mapping')
        self.influencesManualMapping.mirrorMode = True
        self.influencesManualMapping.createUI(manualGroup)
        self.influencesManualMapping.getSelectedInfluences = lambda: self.influenceMappingPreview.currentInfluencesSelection

        def onManualOverridesChanged():
            mll = MllInterface()
            mll.setManualMirrorInfluences(self.influencesManualMapping.manualOverrides)
            self.previewInfluenceMapping()
        self.influencesManualMapping.manualOverridesChanged.addHandler(onManualOverridesChanged, contentRoot)

        def updateForSelectionInfoChange():
            log.info("updating for selection change")
            layersAvailable = selectionState.getLayersAvailable()
            cmds.layout(contentRoot,e=True,enable=layersAvailable)
            if not layersAvailable:
                log.info("layers unavailable")
                return

            self.previewInfluenceMapping()

        selectionState.selectionInfo.changed.addHandler(updateForSelectionInfoChange,contentRoot)
        selectionState.mirrorInfo.changed.addHandler(self.previewInfluenceMapping,contentRoot)
        updateForSelectionInfoChange()

    def previewInfluenceMapping(self):
        log.info("preview influence mapping")
        if self.buildMapper is None:
            log.info("mapper not available")
            return

        if not selectionState.getLayersAvailable():
            log.info("layers not available")
            return

        self.influencesManualMapping.manualOverrides = selectionState.mll.getManualMirrorInfluences()

        self.mapper.sourceInfluences = selectionState.mll.listInfluenceInfo()
        self.mapper = self.buildMapper()

        self.mapper.manualOverrides = self.influencesManualMapping.manualOverrides
        self.mapper.calculate()

        self.influenceMappingPreview.mapper = self.mapper
        self.influenceMappingPreview.constructInfluenceList()


"""inventario URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from django.conf.urls.static import static
from django.conf import settings
from django.conf.urls import url
from django.views.static import serve

from menuAndWelcome.views import inicioView, licenciaView, register
from inventarioTienda.views import (controlVentasView, listarControlVentasView, crearControlVentasView,
                                    companyView, pagoControlVentasView, crearPagoControlVentasView,
                                    pagosView, listarPagosView, crearPagosView, boletaCabeceraFinalView,
                                    crearBoletaCabeceraView, crearBoletaDetTraView, updateBoletaDetTraView,
                                    deleteBoletaDetTraView, crearBoletaDetArtView, updateBoletaDetArtView,
                                    deleteBoletaDetArtView, boletaCabeceraFinalEstView, articulosView,
                                    listarArticulosView, crearArticulosView, editarArticulosView, eliminarArticulosView,
                                    trabajadorView, listarTrabajadoresView, crearTrabajadorView, editarTrabajadorView,
                                    eliminarTrabajadorView, transaccionesView, listarTransaccionesView, crearTransaccionesView,
                                    editarTransaccionesView, eliminarTransaccionesView, clienteView, listarClientesView,
                                    crearClientesView, editarClientesView, eliminarClientesView, impresionView,
                                    deleteBoletaCabView)

urlpatterns = [
    path('', inicioView, name="inicio"),
    path('licencia/', licenciaView, name="licencia"),

    path('controlVentas/<int:index>', controlVentasView, name="controlVentas"),
    path('listarControlVentas/', listarControlVentasView, name="listarControlVentas"),
    path('crearControlVentas/', crearControlVentasView, name="crearControlVentas"),

    path('company/<int:index>', companyView, name="company"),

    path('pagoControl/<int:index>', pagoControlVentasView, name="pagoControlVentas"),
    path('crearPagoControl/', crearPagoControlVentasView, name="crearPagoControlVentas"),

    path('pagos/<int:index>', pagosView, name="pagos"),
    path('listarPagos/', listarPagosView, name="listarPagos"),
    path('crearPagos/', crearPagosView, name="crearPagos"),

    path('boletaCabFin/<int:index>', boletaCabeceraFinalView, name="boletaCabFin"),
    path('crearBoletaCab/<nombre>/<int:dni>', crearBoletaCabeceraView, name="crearBoletaCabFin"),
    path('boletaCabFinEst/<int:index>', boletaCabeceraFinalEstView, name="boletaCabFinEst"),
    path('deleteBoletaCab/<int:index>', deleteBoletaCabView, name="deleteBolCab"),

    path('crearBoletaDetTra/<int:index>/<int:indexTra>', crearBoletaDetTraView, name="crearBoletaDetTra"),
    path('updateBoletaDetTra/<int:index>/<int:indexTra>/<int:cantidad>', updateBoletaDetTraView, name="updateBoletaDetTra"),
    path('deleteBoletaDetTra/<int:index>/<int:indexTra>', deleteBoletaDetTraView, name="deleteBoletaDetTra"),

    path('crearBoletaDetArt/<int:index>/<int:indexArt>', crearBoletaDetArtView, name="crearBoletaDetArt"),
    path('updateBoletaDetArt/<int:index>/<int:indexArt>/<int:cantidad>', updateBoletaDetArtView, name="updateBoletaDetArt"),
    path('deleteBoletaDetArt/<int:index>/<int:indexArt>', deleteBoletaDetArtView, name="deleteBoletaDetArt"),

    path('articulos/<int:index>', articulosView, name="verArticulos"),
    path('listarArticulo/', listarArticulosView, name="listarArticulo"),
    path('crearArticulo/', crearArticulosView, name="crearArticulo"),
    path('editarArticulo/<int:index>', editarArticulosView, name="editarArticulo"),
    path('eliminarArticulo/<int:index>', eliminarArticulosView, name="eliminarArticulo"),

    path('trabajadores/<int:index>', trabajadorView, name="verTrabajador"),
    path('listarTrabajador/', listarTrabajadoresView, name="listarTrabajador"),
    path('crearTrabajador/', crearTrabajadorView, name="crearTrabajador"),
    path('editarTrabajador/<int:index>', editarTrabajadorView, name="editarTrabajador"),
    path('eliminarTrabajador/<int:index>', eliminarTrabajadorView, name="eliminarTrabajador"),

    path('transacciones/<int:index>', transaccionesView, name="verTransaccion"),
    path('listarTransaccion/', listarTransaccionesView, name="listarTransaccion"),
    path('crearTransaccion/', crearTransaccionesView, name="crearTransaccion"),
    path('editarTransaccion/<int:index>', editarTransaccionesView, name="editarTransaccion"),
    path('eliminarTransaccion/<int:index>', eliminarTransaccionesView, name="eliminarTransaccion"),

    path('clientes/<int:index>', clienteView, name="verCliente"),
    path('listarCliente/', listarClientesView, name="listarCliente"),
    path('crearCliente/', crearClientesView, name="crearCliente"),
    path('editarCliente/<int:index>', editarClientesView, name="editarCliente"),
    path('eliminarCliente/<int:index>', eliminarClientesView, name="eliminarCliente"),

    path('imprimir/<int:index>', impresionView, name="eliminarCliente"),

    path('admin/', admin.site.urls),
    path('register', register, name = 'register'),
    path('user/', include("django.contrib.auth.urls"), name = 'users'),
    path('logs/', include('log_viewer.urls')),
    #https://stackoverflow.com/questions/5836674/why-does-debug-false-setting-make-my-django-static-files-access-fail
    url(r'^media/(?P<path>.*)$', serve,{'document_root': settings.MEDIA_ROOT}),
    url(r'^static/(?P<path>.*)$', serve,{'document_root': settings.STATIC_ROOT}),
] + static (settings.STATIC_URL, document_root=settings.STATIC_ROOT)


from django.db import models, transaction
from django.conf import settings
from management.models import Organization
from alerts.models import Channel, AlertStats
from django.core.signing import TimestampSigner
from django.shortcuts import get_object_or_404, reverse
from django.template.loader import render_to_string
from webpush.models import PushInformation, Group
from webpush.utils import send_notification_to_group
import json

# Emails
from django.core.mail import send_mass_mail, send_mail

# SUBSCRIPTION GROUPS
# -------------------

# A SubscriptionGroup is a collection of subscriptions that are
# grouped for (the opportunity of) greater efficiency when
# doing a 'multicast send' (i.e. broadcast). After all, it's
# far more efficient to send one message to many emails, for example,
# than to send the message one email at a time.
class SubscriptionGroup:
    def __init__(self, subscriptions):
        self.subscriptions = subscriptions

    # Get all the subscribers to a particular channel and
    # assemble them into a new SubscriptionGroup that
    # can be used for sending.
    @staticmethod
    def for_channel(channel):
        raise NotImplementedError

    # Push the given alert to every subscription in the
    # SubscriptionGroup. Naive implementations will
    # send the message to each subscription indiv-
    # idually, but better ones will make use of
    # multicast systems (like Django's multi-email
    # send functionality).
    def push(self, alert):
        raise NotImplemented

# EMAIL SUBSCRIPTIONS
# -------------------

class EmailSubscription(models.Model):
    email = models.EmailField()
    organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
    channels = models.ManyToManyField(Channel)
    verified = models.BooleanField(default=False)
    created = models.DateTimeField(auto_now_add=True)

    unique_together = ["email", "organization"]

    def get_verification_token(self):
        return TimestampSigner().sign(str(self.id))

    @staticmethod
    def activate_verification_token(token):
        subscription = get_object_or_404(
            EmailSubscription, id=TimestampSigner().unsign(token, max_age=24 * 60 * 60)
        )
        subscription.verified = True
        subscription.save()
        return subscription

    def get_access_token(self):
        return TimestampSigner().sign(str(self.id))

    @staticmethod
    def for_access_token(token):
        return get_object_or_404(
            EmailSubscription,
            id=TimestampSigner().unsign(token, max_age=30 * 24 * 60 * 60),
        )

    def send_opt_in(self):
        link = "/TODO"
        send_mail(
            "Confirm email sign-up",
            render_to_string(
                "subscriptions/email/opt-in.txt",
                context={
                    "link": link,
                    "organization": self.organization,
                    "subscription": self,
                    "link_prefix": settings.PROTOCOL_AND_HOST,
                },
            ),
            f"{self.organization.name} <{settings.EMAIL_FROM}>",
            [self.email],
        )

class EmailSubscriptionGroup(SubscriptionGroup):
    @staticmethod
    def for_channel(channel):
        return EmailSubscriptionGroup(
            EmailSubscription.objects.filter(channels__in=[channel], verified=True)
        )

    def push(self, alert):
        # TODO: send nice-looking HTML messages, potentially with
        # the news organization's logo
        emails = [
            (  # TODO: design nice message template
                f"{alert.channel.organization.name}: {alert.channel.name}",
                render_to_string(
                    "subscriptions/email/alert.txt",
                    context={
                        "alert": alert,
                        "alert_url": alert.get_public_url("em"),
                        "organization": subscription.organization,
                        "subscription": subscription,
                        "link_prefix": settings.PROTOCOL_AND_HOST,
                    },
                ),
                f"{alert.channel.organization.name} <{settings.EMAIL_FROM}>",
                [subscription.email],
            )
            for subscription in self.subscriptions
        ]
        send_mass_mail(emails)

        with transaction.atomic():
            stats = AlertStats.for_alert_locking(alert)
            stats.email_sends += len(self.subscriptions)
            stats.save()

# WEBPUSH SUBSCRIPTIONS
# ---------------------

class WebpushSubscription(models.Model):
    organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
    subscription = models.ForeignKey(PushInformation, on_delete=models.CASCADE)
    created = models.DateTimeField(auto_now_add=True)

class WebpushSubscriptionGroup(SubscriptionGroup):
    def __init__(self, group):
        self.group = group
        super(WebpushSubscriptionGroup, self).__init__(None)

    @staticmethod
    def for_channel(channel):
        group = Group.objects.get(name=str(channel.organization.id))
        return WebpushSubscriptionGroup(group)

    def push(self, alert):
        send_notification_to_group(
            self.group.name,
            payload=json.dumps(
                {
                    "head": f"{alert.channel.name} / {alert.channel.organization.name}",  # TODO: make configurable
                    "body": alert.content,
                    "url": settings.PROTOCOL_AND_HOST + alert.get_public_url("wp"),
                }
            ),
            ttl=1000,
        )
        with transaction.atomic():
            stats = AlertStats.for_alert_locking(alert)
            stats.webpush_sends += WebpushSubscription.objects.filter(
                organization=alert.channel.organization
            ).count()
            stats.save()


#!/usr/bin/env python

import numpy as np
# Dataset
n_samples, n_features = 100, 2
mean0, mean1 = np.array([0, 0]), np.array([0, 2])
Cov = np.array([[1, .8],[.8, 1]])
np.random.seed(42)
X0 = np.random.multivariate_normal(mean0, Cov, n_samples)
X1 = np.random.multivariate_normal(mean1, Cov, n_samples)
X = np.vstack([X0, X1])
y = np.array([0] * X0.shape[0] + [1] * X1.shape[0])

"""
Ridge linear Support Vector Machine (L2-regularization)
Support Vector Machine seek for separating hyperplane with maximum margin to enforce robustness against noise.
Like logistic regression it is a discriminative method that only focuses of predictions.
Here we present the non separable case of Maximum Margin Classifiers with ±1 coding (ie.: y i {−1, +1})
So linear SVM is closed to Ridge logistic regression, using the hinge loss instead of the logistic loss. Both will provide
very similar predictions.
"""

from sklearn import svm
svmlin = svm.LinearSVC()
# Remark: by default LinearSVC uses squared_hinge as loss
svmlin.fit(X, y)
y_pred_svmlin = svmlin.predict(X)
errors = y_pred_svmlin != y
print("Nb errors=%i, error rate=%.2f" % (errors.sum(), errors.sum() / len(y_pred_svmlin)))
print(svmlin.coef_)

"""Lasso linear Support Vector Machine (L1-regularization)
Linear SVM for classification (also called SVM-C or SVC) with l1-regularization
∑ N
min F Lasso linear SVM (w) = λ ||w|| 1 + Ci ξ i

with ∀i y i (w · x i ) ≥ 1 − ξ i
"""
svmlinl1 = svm.LinearSVC(penalty='l1', dual=False)
# Remark: by default LinearSVC uses squared_hinge as loss
svmlinl1.fit(X, y)
y_pred_svmlinl1 = svmlinl1.predict(X)
errors = y_pred_svmlinl1 != y
print("Nb errors=%i, error rate=%.2f" % (errors.sum(), errors.sum() / len(y_pred_svmlinl1)))
print(svmlinl1.coef_)


# Advent of Code 2022 Day 02
# Author: Erik Båvenstrand
# URL: https://adventofcode.com/2022/day/2

from typing import Union

def part_a(data: list[str]) -> Union[int, str, None]:
    """Solution to part A.

    Args:
        data (list[str]): Advent of Code challenge input.

    Returns:
        Union[int, str, None]: Solution to the challenge.
    """
    score = 0
    for moves in data:
        elf_move = ord(moves[0]) - 65
        player_move = ord(moves[2]) - 88
        if player_move == elf_move:
            score += player_move + 4
        elif player_move == (elf_move + 1) % 3:
            score += player_move + 7
        elif player_move == (elf_move - 1) % 3:
            score += player_move + 1
    return score

def part_b(data: list[str]) -> Union[int, str, None]:
    """Solution to part B.

    Args:
        data (list[str]): Advent of Code challenge input.

    Returns:
        Union[int, str, None]: Solution to the challenge.
    """
    score = 0
    for moves in data:
        elf_move = ord(moves[0]) - 65
        outcome = moves[2]
        if outcome == "X":
            score += 1 + (elf_move - 1) % 3
        elif outcome == "Y":
            score += 4 + elf_move
        elif outcome == "Z":
            score += 7 + (elf_move + 1) % 3
    return score