"""Tests for the ROI and StructureSet classes."""

import fnmatch
import math
import os
import random
import shutil
import pandas as pd
import pytest
import numpy as np
import matplotlib.colors

from shapely.geometry import Polygon
from shapely.validation import explain_validity

from skrt.simulation import SyntheticImage
from skrt.structures import contour_to_polygon, polygon_to_contour, \
        StructureSet, ROI

# Make temporary test dir
if not os.path.exists("tmp"):
    os.mkdir("tmp")

# Create synthetic structure set
sim = SyntheticImage((100, 100, 40))
sim.add_cube(side_length=40, name="cube", intensity=1)
sim.add_sphere(radius=20, name="sphere", intensity=10)
structure_set = sim.get_structure_set()
cube = structure_set.get_roi("cube")

def test_structure_set_from_sythetic_image():
    assert isinstance(structure_set, StructureSet)

def test_write_nifti():
    nii_dir = "tmp/nii_structs"
    if os.path.exists(nii_dir):
        shutil.rmtree(nii_dir)
    structure_set.write(outdir="tmp/nii_structs")
    assert len(os.listdir(nii_dir)) == 2

def test_write_txt():
    txt_dir = "tmp/txt"
    if os.path.exists(txt_dir):
        shutil.rmtree(txt_dir)
    structure_set.write(outdir="tmp/txt", ext='.txt')
    assert len(os.listdir(txt_dir)) == 2

def test_get_rois():
    assert len(structure_set.get_rois()) == 2

def test_get_roi_names():
    names = structure_set.get_roi_names()
    assert len(names) == 2
    assert "cube" in names
    assert "sphere" in names

def test_get_dict():
    sdict = structure_set.get_roi_dict()
    assert len(sdict) == 2
    assert set(sdict.keys()) == set(["cube", "sphere"])

def test_get_roi():
    assert isinstance(cube, ROI)
    assert cube.name == "cube"
    # ROI from non-DICOM source
    assert cube.number == None

def test_structure_set_from_rois():
    """Test creation of structure set from ROIs."""

    structure_set2 = StructureSet(structure_set.get_rois())
    assert isinstance(structure_set2, StructureSet)
    sdict1 = structure_set.get_roi_dict()
    sdict2 = structure_set2.get_roi_dict()
    assert len(sdict1.keys()) > 0
    for key in sdict1:
        assert key in sdict2

def test_rename():
    new_names = {
        "cube2": ["cube", "test"],
        "sphere2": "spher*"
    }
    structure_set.rename_rois(new_names)
    assert structure_set.get_rois()[0].name != structure_set.get_rois()[0].original_name
    assert set(structure_set.get_roi_names()) == set(new_names.keys())

    # Check that original names are kept
    n_match = 0
    for roi in structure_set:
        if roi.name in new_names:
            match = False
            for old_name in new_names[roi.name]:
                if fnmatch.fnmatch(roi.original_name.lower(), old_name.lower()):
                    match = True
                    n_match += 1
                    break
            assert match
    assert n_match == len(new_names.keys())

    old_names = {
        "cube": "cube2",
        "sphere": "sphere2"
    }
    structure_set.rename_rois(old_names)
    assert set(structure_set.get_roi_names()) == set(old_names.keys())

def test_copy_rename():
    """Test copying with ROI renaming."""

    new_names = {"cube3": "cube"}
    structure_set2 = structure_set.filtered_copy(new_names, name="copy", keep_renamed_only=True)
    assert len(structure_set2.get_rois()) == 1
    assert structure_set2.get_roi_names() == ["cube3"]
    assert structure_set2.name == "copy"
    structure_set.rename_rois({"cube": "cube3"})

def test_copy_remove():
    """Test copying with ROI removal."""

    structure_set2 = structure_set.filtered_copy(to_remove="cube")
    assert structure_set2.get_roi_names() == ["sphere"]

def test_copy_keep():
    """Test copying with keeping only certain ROIs."""

    structure_set2 = structure_set.filtered_copy(to_keep="sphere")
    assert structure_set2.get_roi_names() == ["sphere"]

def test_clone():
    """Test cloning; check that ROIs are fully copied."""

    sphere1 = structure_set.get_roi("sphere")
    sphere1.create_mask()
    structure_set2 = structure_set.clone()
    sphere2 = structure_set2.get_roi("sphere")
    sphere1.set_color("red")
    sphere2.set_color("blue")
    assert sphere1.color != sphere2.color
    assert sphere1 is not sphere2
    sphere2.transform(translation=(3, 3, 0))
    assert not np.all(sphere1.get_mask() == sphere2.get_mask())

def test_init_from_structure_set():
    sphere1 = structure_set.get_roi("sphere")
    sphere1.create_mask()
    structure_set2 = StructureSet(structure_set)
    sphere2 = structure_set2.get_roi("sphere")
    sphere1.set_color("red")
    sphere2.set_color("blue")
    assert sphere1.color != sphere2.color
    assert sphere1 is not sphere2
    sphere2.transform(translation=(3, 3, 0))
    assert not np.all(sphere1.get_mask() == sphere2.get_mask())

def test_filtered_copy_no_copy_data():
    """Test copying but using same data for ROIs."""

    # First check that data is copied correctly
    structure_set2 = structure_set.filtered_copy(copy_roi_data=True)
    assert structure_set2.get_rois()[0].get_mask() \
            is not structure_set.get_rois()[0].get_mask()

    # Check that data is passed by reference but new ROI object is made
    structure_set2 = structure_set.filtered_copy(copy_roi_data=False)
    assert structure_set2.get_rois()[0] is not structure_set.get_rois()[0]
    assert structure_set2.get_rois()[0].get_mask() \
            is structure_set.get_rois()[0].get_mask()

def test_read_nii():
    nii_dir = "tmp/nii_structs"
    structs_from_nii = StructureSet(nii_dir)
    assert len(structs_from_nii.get_rois()) == 2
    assert set(structure_set.get_roi_names()) \
            == set(structs_from_nii.get_roi_names())

def test_plot_contour():
    cube.plot(plot_type="contour", show=False)

def test_plot_centroid():
    cube.plot(plot_type="centroid", show=False)

def test_plot_mask():
    cube.plot(plot_type="mask", show=False)

def test_plot_filled():
    cube.plot(plot_type="filled", show=False)

def test_get_geometry():
    geom = structure_set.get_geometry()
    assert isinstance(geom, pd.DataFrame)
    assert geom.shape[0] == 2

def test_get_comparison_pairs():
    """Get comparison pairs with self; should return each ROI paired with every
    other except itself."""

    pairs = structure_set.get_comparison_pairs()
    n_rois = len(structure_set.get_rois())
    assert len(pairs) == n_rois ** 2 - n_rois
    assert len(pairs[0]) == 2

def test_get_comparison_pairs_with_other():
    """Get comparison pairs with another StructureSet; should return each ROI
    paired with each ROI with a matching names."""

    structure_set2 = StructureSet("tmp/nii_structs")
    pairs = structure_set.get_comparison_pairs(structure_set2)
    assert len(pairs) == 2
    assert pairs[0][0].name == pairs[0][1].name
    assert pairs[1][0].name == pairs[1][1].name

def test_get_comparison_pairs_all():
    """Get comparison pairs with another StructureSet using the 'all' method;
    should return every ROI in one set paired with every ROI in the other,
    regardless of name match."""

    structure_set2 = StructureSet("tmp/nii_structs")
    pairs = structure_set.get_comparison_pairs(structure_set2, "all")
    assert len(pairs) == len(structure_set.get_rois()) \
            * len(structure_set2.get_rois())

def test_compare_with_own_consensus():
    """Get comparison of each ROI with consensus of all others."""

    pairs = structure_set.get_comparison_pairs(comp_type="consensus",
                                               consensus_type="sum")
    assert len(pairs) == len(structure_set.get_rois())
    assert pairs[0][1] == structure_set.get_sum(exclude=pairs[0][0].name)

def test_compare_with_other_consensus():
    """Get comparison of each ROI with consensus of another structure set."""

    structure_set2 = StructureSet("tmp/nii_structs")
    pairs = structure_set.get_comparison_pairs(structure_set2,
                                               comp_type="consensus",
                                               consensus_type="sum")
    assert len(pairs) == len(structure_set.get_rois())
    assert pairs[0][1] == structure_set2.get_sum()
    assert pairs[1][1] == structure_set2.get_sum()

def test_plot_consensus():
    to_exclude = structure_set.get_rois()[-1].name
    for include_image in [True, False]:
        structure_set.plot(consensus_type="overlap",
                           exclude_from_consensus=to_exclude,
                           show=False,
                           include_image=include_image)

def test_get_comparison():
    comp = structure_set.get_comparison()
    assert isinstance(comp, pd.DataFrame)
    assert comp.shape[0] == 2

def test_plot_comparisons():
    plot_dir = "tmp/struct_plots"
    if os.path.exists(plot_dir):
        shutil.rmtree(plot_dir)
    structure_set2 = StructureSet("tmp/nii_structs")
    structure_set2.plot_comparisons(outdir=plot_dir, show=False)
    assert len(os.listdir(plot_dir)) == 2

def compare_rois(roi0, roi1):
    '''Compare two rois.'''

    # Check basic characteristics
    assert roi0 is not roi1
    assert roi0.color == roi1.color
    assert roi0.get_centroid().all() == roi1.get_centroid().all()

    # Check number of planes.
    roi0_keys = list(roi0.get_contours().keys())
    roi1_keys = list(roi1.get_contours().keys())
    assert len(roi0_keys) == len(roi1_keys)

    roi0_keys.sort()
    roi1_keys.sort()

    for i in range(len(roi0_keys)):

        # Check plane z-coordinates.
        assert roi0_keys[i] == roi1_keys[i]

        # Check number of contours in plane.
        contours0 = roi0.get_contours()[roi0_keys[i]]
        contours1 = roi1.get_contours()[roi1_keys[i]]
        assert len(contours0) == len(contours1)

        # Check contour points.
        for j in range(len(contours0)):
            assert contours0[j].all() == contours1[j].all()

def test_write_dicom():
    '''Test writing of structure set to dicom.'''
    dcm_dir = "tmp/dcm_structs"
    if os.path.exists(dcm_dir):
        shutil.rmtree(dcm_dir)
    structure_set.write(outdir=dcm_dir, ext='dcm')
    assert len(os.listdir(dcm_dir)) == 1

def test_dicom_dataset():
    '''Check that structure set written to dicom matches original.'''
    dcm_dir = "tmp/dcm_structs1"
    if os.path.exists(dcm_dir):
        shutil.rmtree(dcm_dir)
    structure_set.write(outdir=dcm_dir, ext='dcm')
    structure_set_dcm = StructureSet(dcm_dir)
    assert structure_set.get_roi_names() == structure_set_dcm.get_roi_names()

    for name in structure_set.get_roi_names():
        roi0 = structure_set.get_roi(name)
        roi1 = structure_set_dcm.get_roi(name)
        compare_rois(roi0, roi1)

def test_write_roi_to_dicom():
    '''Check that individual rois written to dicom match originals.'''
    dcm_dir = "tmp/dcm_structs2"
    if os.path.exists(dcm_dir):
        shutil.rmtree(dcm_dir)
    rois = list(structure_set.get_rois())
    assert len(rois) == 2

    for i in range(len(rois)):
        rois[i].write(outdir=dcm_dir, ext='dcm', overwrite=True)
        assert len(os.listdir(dcm_dir)) == 1

        structure_set_dcm = StructureSet(dcm_dir)
        rois_dcm = list(structure_set_dcm.get_rois())

        assert len(rois_dcm) == 1

        compare_rois(rois[i], rois_dcm[0])

def test_write_roi_with_source_to_dicom():
    '''Check that adding rois to dicom file gives original structure set.'''
    dcm_dir = "tmp/dcm_structs3"
    if os.path.exists(dcm_dir):
        shutil.rmtree(dcm_dir)
    rois = list(structure_set.get_rois())
    assert len(rois) == 2

    rois[0].write(outdir=dcm_dir, ext='dcm', overwrite=True)
    assert len(os.listdir(dcm_dir)) == 1
    rois[1].write(outdir=dcm_dir, ext='dcm', header_source=dcm_dir,
            overwrite=False)
    filenames = os.listdir(dcm_dir)
    filenames.sort()
    assert len(filenames) == 2

    dcm_path = os.path.join(dcm_dir, filenames[-1])
    structure_set_dcm = StructureSet(dcm_path)
    rois_dcm = list(structure_set_dcm.get_rois())

    assert len(rois_dcm) == 2

    for name in structure_set.get_roi_names():
        roi0 = structure_set.get_roi(name)
        roi1 = structure_set_dcm.get_roi(name)
        compare_rois(roi0, roi1)

def test_init_from_roi():
    sphere1 = structure_set.get_roi("sphere")
    sphere2 = ROI(sphere1)
    sphere1.set_color("red")
    sphere2.set_color("blue")
    assert sphere1.color != sphere2.color

def test_roi_from_image_threshold():
    roi = ROI(sim.get_image(), mask_threshold=5)
    assert roi.get_area() == sim.get_roi("sphere").get_area()

def test_roi_no_image_with_geom():
    """Check that an ROI object can be created from contours only plus
    geometric info."""

    new_roi = ROI(cube.get_contours(),
                  origin=cube.get_origin(),
                  voxel_size=cube.get_voxel_size(),
                  shape=cube.get_mask().shape
                 )
    assert np.all(new_roi.get_mask() == cube.get_mask())

def test_roi_from_polygons():
    """Check that an ROI object can be created from shapely polygons."""

    new_roi = ROI(cube.get_polygons(),
                  origin=cube.get_origin(),
                  voxel_size=cube.get_voxel_size(),
                  shape=cube.get_mask().shape
                 )
    assert np.all(new_roi.get_mask() == cube.get_mask())

    df = StructureSet(new_roi).get_comparison(StructureSet(cube))
    # Dice score may not be exactly 1 because of rounding errors
    assert df['dice'][0] == pytest.approx(1.0, abs=1.e-3)

def test_null_roi():
    roi = ROI()
    assert(type(roi).__name__ == "ROI")
    assert(roi.affine is None)
    assert(roi.contours == {})
    assert(roi.custom_color is False)
    assert(roi.image is None)
    assert(roi.input_contours == {})
    assert(roi.kwargs == {})
    assert(roi.contours == {})
    assert(roi.loaded_mask is False)
    assert(roi.origin is None)
    assert(roi.original_name is None)
    assert(roi.source_type is None)
    assert(roi.shape is None)
    assert(roi.source is None)
    assert(roi.title is None)
    assert(roi.voxel_size is None)

def test_null_structure_set():
    ss = StructureSet()
    assert(ss.date == "")
    assert(ss.files == [])
    assert(ss.image is None)
    assert(ss.loaded is True)
    assert(ss.multi_label is False)
    assert(ss.names is None)
    assert(ss.path == "")
    assert(ss.rois == [])
    assert(ss.sources == [])
    assert(ss.subdir == "")
    assert(ss.time == "")
    assert(ss.timestamp =="")
    assert(ss.to_keep is None)
    assert(ss.to_remove is None)

def test_contour_to_polygon():
    # Create self-intersecting polygon
    contour = [(0, 0), (0, 2), (2, 2), (1.9, 2.1), (1.9, 0), (0, 0)]
    p1 = Polygon(contour)
    assert 'self-intersection' in explain_validity(p1).lower()
    p2 = contour_to_polygon(contour)
    assert 'self-intersection' not in explain_validity(p2).lower()

def test_contour_to_polygon_to_contour():
    '''Test converting from contour to polygon and back.'''
    for key, contours in cube.get_contours().items():
        for contour1 in contours:
            polygon = contour_to_polygon(contour1)
            contour2 = polygon_to_contour(polygon)
            n_point1 = len(contour1)
            n_point2 = len(contour2)
            assert n_point1 == n_point2
            assert contour1.all() == contour2.all()

def test_dummy_image():
    """Test setting an ROI's image to a dummy image with a given shape."""

    roi2 = cube.clone()
    nx = [20, 200]
    ny = [30, 150]
    for x, y in zip(nx, ny):
        roi2.set_image_to_dummy(shape=(x, y))
        assert roi2.image.get_data().shape[1] == x
        assert roi2.image.get_data().shape[0] == y

def test_dummy_image_from_voxel_size():
    """Test setting an ROI's image to a dummy image with given voxel sizes."""

    vx = 1.5
    vy = 1
    roi2 = cube.clone()
    roi2.set_image_to_dummy(voxel_size=(vx, vy))
    assert roi2.image.voxel_size[0] == vx
    assert roi2.image.voxel_size[1] == vy

def test_forced_mask_recreation():
    """Recreate a contour ROI's mask with a given shape or voxel size."""

    roi2 = ROI(cube.get_contours("x-y"))
    nx = [20, 200]
    ny = [30, 150]
    for x, y in zip(nx, ny):
        roi2.create_mask(force=True, shape=(x, y))
        assert roi2.get_mask().shape[1] == x
        assert roi2.get_mask().shape[0] == y

    vx = 1.5
    vy = 1
    roi2.create_mask(force=True, voxel_size=(vx, vy))
    assert roi2.mask.get_voxel_size()[0] == vx
    assert roi2.mask.get_voxel_size()[1] == vy

def test_overlap_level():
    """Create mask from contours using different overlap requirements."""

    # First set tight overlap level in __init__
    roi = ROI(sim.get_roi("sphere").get_contours(), overlap_level=1)
    roi.create_mask()

    # Mask area should be smaller than contour area with this requirement
    assert roi.get_area(method="mask") < roi.get_area(method="contour")

    # Force mask recreation with overlap_level set in create_mask()
    level = 0.1
    prev_mask = roi.get_mask()
    roi.create_mask(force=True, overlap_level=level)
    assert not np.all(roi.get_mask() == prev_mask)
    assert roi.overlap_level == level

    # Mask area should be larger than contour area with loose overlap
    assert roi.get_area(method="mask") > roi.get_area(method="contour")

def test_slice_number_conversions():
    """For an ROI with no image, check position-contour-index conversion works."""
    roi2 = ROI(cube.get_contours("x-y"))
    assert roi2.pos_to_idx(roi2.idx_to_pos(0, "z"), "z") == 0
    assert roi2.pos_to_slice(roi2.slice_to_pos(1, "z"), "z") == 1
    assert roi2.slice_to_idx(roi2.idx_to_slice(0, "z"), "z") == 0

def test_slice_number_conversions_image():
    """For an ROI with an image, check position-contour-index conversion works."""

    roi2 = ROI(cube.get_contours("x-y"))
    roi2.set_image_to_dummy()
    im = roi2.image
    assert roi2.pos_to_idx(0, "z") == im.pos_to_idx(0, "z")
    assert roi2.idx_to_pos(0, "z") == im.idx_to_pos(0, "z")
    assert roi2.pos_to_slice(0, "z") == im.pos_to_slice(0, "z")
    assert roi2.slice_to_pos(0, "z") == im.slice_to_pos(0, "z")
    assert roi2.slice_to_idx(0, "z") == im.slice_to_idx(0, "z")
    assert roi2.idx_to_slice(0, "z") == im.idx_to_slice(0, "z")

# Create structure set for transform tests
sim0 = SyntheticImage((100, 100, 40))
sim0.add_sphere(radius=5, centre=(20, 70, 30), name="sphere", intensity=10)
sim0.add_cube(side_length=5, centre=(30, 20, 10), name="cube", intensity=1)
ss0 = sim0.get_structure_set()

# Set limits for random-number generation for transform tests
dx_min, dx_max = (-15, 65)
dy_min, dy_max = (-15, 25)
xyc_min, xyc_max = (40, 60)
zc_min, zc_max = (15, 25)
dz_min, dz_max = (-5, 5)
theta_min, theta_max = (0, 360)

# Set level of agreement (mm) for post-transform centroid positions
precision = 0.5

def test_structure_set_translation():
    '''Test structure-set translation.'''
    n_test = 10
    random.seed(1)

    for i in range(n_test):
        # Randomly translate structure sets.
        dx = random.uniform(dx_min, dx_max)
        dy = random.uniform(dy_min, dy_max)
        dz = random.uniform(dz_min, dz_max)
        translation = [dx, dy, dz]
        # Translation using masks.
        ss1 = ss0.clone()
        ss1.transform(translation=translation)
        # Translation using contours.
        ss2 = ss0.clone()
        ss2.transform(translation=translation, force_contours=True)

        # Calculate expected centroid, and compare with transform results.
        for roi in ss0.get_rois():
            translated_centroid = [roi.get_centroid()[i] + translation[i]
                for i in range(3)]

            roi1 = ss1.get_roi(roi.name)
            roi2 = ss2.get_roi(roi.name)
            for i in range(3):
                assert roi1.get_centroid()[i] == pytest.approx(
                        translated_centroid[i], abs=precision)
                assert roi2.get_centroid()[i] == pytest.approx(
                        translated_centroid[i], abs=precision)

def test_structure_set_rotation():
    '''Test structure-set rotation.'''
    n_test = 10
    random.seed(1)

    for i in range(n_test):
        # Randomly rotate structure sets.
        xc = random.uniform(xyc_min, xyc_max)
        yc = random.uniform(xyc_min, xyc_max)
        zc = random.uniform(zc_min, zc_max)
        centre = (xc, yc, zc)
        angle = random.uniform(theta_min, theta_max)
        rad_angle = math.radians(angle)
        rotation = [0, 0, angle]
        # Rotation using masks.
        ss1 = StructureSet(ss0)
        ss1.transform(centre=centre, rotation=rotation)
        # Rotation using contours.
        ss2 = StructureSet(ss0)
        ss2.transform(centre=centre, rotation=rotation, force_contours=True)

        # Calculate expected centroid, and compare with transform results.
        for roi in ss0.get_rois():
            x0, y0, z0 = roi.get_centroid()
            dx = x0 - xc
            dy = y0 - yc
            xr = dx * math.cos(rad_angle) - dy * math.sin(rad_angle) + xc
            yr = dx * math.sin(rad_angle) + dy * math.cos(rad_angle) + yc
            zr = z0
            rotated_centroid = [xr, yr, zr]

            roi1 = ss1.get_roi(roi.name)
            roi2 = ss2.get_roi(roi.name)
            for i in range(3):
                assert roi1.get_centroid()[i] == pytest.approx(
                        rotated_centroid[i], abs=precision)
                assert roi2.get_centroid()[i] == pytest.approx(
                        rotated_centroid[i], abs=precision)

def test_get_extent():
    sim = SyntheticImage((10, 10, 10), origin=(0.5, 0.5, 0.5))
    roi = sim.add_cuboid((4, 2, 6), name="cube")
    ext = roi.get_extents()
    assert ext[0] == [3, 7]
    assert ext[1] == [4, 6]
    assert ext[2] == [2, 8]
    ext = roi.get_extents(buffer=2)
    assert ext[0] == [1, 9]
    assert ext[1] == [2, 8]
    assert ext[2] == [0, 10]

# Make structure set with nearby structs for consensus calculation
sim2 = SyntheticImage((50, 50, 10))
sim2.add_sphere(radius=5, centre=(25, 25, 5), name="sphere1")
sim2.add_sphere(radius=5, centre=(26, 26, 5), name="sphere2")
ss2 = sim.get_structure_set()

def test_get_staple():
    staple = ss2.get_staple()
    assert 0 < staple.get_volume() < sum([roi.get_volume()
                                          for roi in ss2.get_rois()])
def test_get_majority_vote():
    mv = ss2.get_majority_vote()
    assert 0 < mv.get_volume() < sum([roi.get_volume()
                                          for roi in ss2.get_rois()])

def test_get_overlap():
    overlap = ss2.get_overlap()
    overlap_mask = ss2.get_rois()[0].get_mask()
    for roi in ss2.get_rois()[1:]:
        overlap_mask *= roi.get_mask()
    assert overlap_mask.sum() == overlap.get_volume(units="voxels")

def test_get_sum():
    overlap = ss2.get_overlap()
    roi_sum = ss2.get_sum()
    all_vol = sum([roi.get_volume() for roi in ss2.get_rois()])
    assert roi_sum.get_volume() == all_vol  - overlap.get_volume()

def test_multi_label():
    """Test reading a StructureSet from a single multi-label array."""

    array = np.zeros(structure_set.get_rois()[0].get_mask().shape)
    for i, roi in enumerate(structure_set):
        array[roi.get_mask()] = i + 1
    ss_multi = StructureSet(array, multi_label=True,
                            names=structure_set.get_roi_names())
    assert len(ss_multi.get_rois()) == len(structure_set.get_rois())
    assert ss_multi.get_roi_names() == structure_set.get_roi_names()

#  def test_recolor():
    #  """Test recoloring ROIs with a list as they are loaded in."""

    #  colors = ["lime", "blue", "orange", "purple", "red"]
    #  ss = StructureSet("tmp/nii_structs", colors=colors)
    #  for i, roi in enumerate(ss.get_rois()):
        #  assert roi.color == matplotlib.colors.to_rbga(colors[i])

#  def test_recolor_from_dict():
    #  """Test recolor based on ROI names."""

    #  sim = SyntheticImage((50, 50, 10))
    #  sim.add_sphere(5, name="sphere1")
    #  sim.add_sphere(10, name="sphere2")
    #  colors = {"*1": "blue", "*2": "red"}
    #  ss = sim.get_structure_set()
    #  ss.recolor_rois(colors)
    #  assert ss.get_roi("sphere1").color == matplotlib.colors.to_rgba("blue")
    #  assert ss.get_roi("sphere2").color == matplotlib.colors.to_rgba("red")


import pytest

from tenable_io.api.models import Group, GroupList

@pytest.mark.vcr()
def test_groups_create(new_group):
    assert isinstance(new_group, Group), u'The `create` method did not return type `Group`.'

@pytest.mark.vcr()
def test_groups_list(client):
    group_list = client.groups_api.list()
    assert isinstance(group_list, GroupList), u'The `list` method did not return type `GroupList`.'

@pytest.mark.vcr()
def test_groups_edit(client, new_group):
    group = new_group
    assert client.groups_api.edit(group.id, 'test_group_edited'), u'The group name was not edited.'
    group_list = client.groups_api.list()
    edited_group = [g for g in group_list.groups if g.id == group.id]
    assert edited_group[0].name == 'test_group_edited', u'The returned group name should match the the edited value.'

@pytest.mark.vcr()
def test_group_users(client, new_group):
    # test will add, list, and remove users from a group
    group = new_group
    user_list = client.users_api.list()

    assert len(user_list.users) > 0, u'User list has at least one user for testing.'
    user = user_list.users[0]

    assert hasattr(user, 'id'), u'User has ID.'
    client.groups_api.add_user(group.id, user.id)

    user_list = client.groups_api.list_users(group.id)
    assert len([u for u in user_list.users if u.id == user.id]) > 0, \
        u'The user returned from group list matches user added to the group.'

    client.groups_api.delete_user(group.id, user.id)

    new_user_list = client.groups_api.list_users(group.id)
    assert len([u for u in new_user_list.users if u.id == user.id]) == 0, \
        u'The removed user is not shown in the group list.'


"""Date field."""
from datetime import date
from typing import Any, Literal, Optional, Union

import attr
from attrs import frozen, validators
from oes.interview.config.field import AskField, FieldBase
from oes.interview.parsing.location import Location
from oes.template import Template

@frozen
class DateAskField(AskField):
    """An :class:`AskField` for a date."""

    type: Literal["date"] = "date"
    optional: bool = False
    default: Any = None
    label: Optional[str] = None
    min: Optional[date] = None
    max: Optional[date] = None
    require_value: Optional[date] = None
    require_value_message: Optional[str] = None

@frozen
class DateField(FieldBase):
    """A date field."""

    type: Literal["date"] = "date"
    set: Optional[Location] = None
    optional: bool = False

    default: Optional[Union[date, Literal["today"]]] = None
    """A default value, or ``"today"`` to use the current date."""

    label: Optional[Template] = None

    require_value: Optional[date] = None
    require_value_message: Optional[str] = None

    min: Optional[Union[date, Literal["today"]]] = None
    """A minimum value, or ``"today"`` to use the current date."""

    max: Optional[Union[date, Literal["today"]]] = None
    """A maximum value, or ``"today"`` to use the current date."""

    def get_python_type(self) -> object:
        return date

    def _get_date(self, val: Optional[Union[date, Literal["today"]]]) -> Optional[date]:
        if val is None:
            return None
        elif val == "today":
            return date.today()
        else:
            return val

    def _validate_min(self, i, a, v):
        min_ = self._get_date(self.min)
        if min_ is not None and v < min_:
            raise ValueError(f"{a.name}: must be on or after {min_}")

    def _validate_max(self, i, a, v):
        max_ = self._get_date(self.max)
        if max_ is not None and v > max_:
            raise ValueError(f"{a.name}: must be on or before {max_}")

    def get_field_info(self) -> Any:
        return attr.ib(
            type=self.get_optional_type(),
            validator=[
                self.validate_required,
                validators.optional(
                    [
                        validators.instance_of(date),
                        self._validate_min,
                        self._validate_max,
                    ]
                ),
            ],
        )

    def get_ask_field(self, context: dict[str, Any]) -> AskField:
        return DateAskField(
            type="date",
            optional=self.optional,
            default=self._get_date(self.default),
            label=self.label.render(**context) if self.label else None,
            min=self._get_date(self.min),
            max=self._get_date(self.max),
            require_value=self.require_value,
            require_value_message=self.require_value_message,
        )


import sqlite3
import data.config

class ColumnsDB:
    def __init__(self, path_to_db=data.config.path_to_db):
        self.path_to_db = path_to_db
        self.create_table_of_columns()

    @property
    def connection(self):
        return sqlite3.connect(self.path_to_db)

    def execute(self, sql: str, parameters: tuple = None, fetchone=False, fetchall=False, commit=False):
        if not parameters:
            parameters = tuple()

        connection = self.connection
        # connection.set_trace_callback(logger)
        cursor = connection.cursor()
        cursor.execute(sql, parameters)
        data = None

        if commit:
            connection.commit()
        if fetchone:
            data = cursor.fetchone()
        if fetchall:
            data = cursor.fetchall()
        connection.close()

        return data

    def create_table_of_columns(self):
        sql = '''
            CREATE TABLE IF NOT EXISTS Columns(
                id integer PRIMARY KEY AUTOINCREMENT,
                desk_id integer,
                name text,
                sequence_number integer,
                FOREIGN KEY (desk_id) REFERENCES Desks(id)
        );
        '''
        self.execute(sql, commit=True)

    def add_column(self, desk_id, name, sequence_number):
        sql = "INSERT INTO Columns(desk_id, name, sequence_number) VALUES(?, ?, ?)"
        parameters = (desk_id, name, sequence_number)
        self.execute(sql, parameters=parameters, commit=True)
        return True

    def select_all_columns(self):
        sql = 'SELECT * FROM Columns'
        return self.execute(sql, fetchall=True)

    @staticmethod
    def format_args(sql, parameters: dict):
        sql += ' AND '.join([
            f"{item} = ?" for item in parameters.keys()
        ])
        return sql, tuple(parameters.values())

    def select_column(self, **kwargs):
        sql = 'SELECT * FROM Columns WHERE '
        sql, parameters = self.format_args(sql, kwargs)
        return self.execute(sql, parameters, fetchone=True)

    def select_columns_by_desk_id(self, desk_id):  # TODO
        sql = 'SELECT * FROM Columns WHERE desk_id=?'
        return self.execute(sql, (desk_id,), fetchall=True)

    def select_column_by_column_id(self, column_id):
        sql = 'SELECT * FROM Columns WHERE id=?'
        return self.execute(sql, (column_id,), fetchone=True)

    def count_columns(self):
        return self.execute("SELECT COUNT(*) FROM Columns;", fetchone=True)[0]

    def update_any_info_about_column(self, column_id, thing_to_change, new_data):  # TODO
        result = self.select_column(id=column_id)
        if not result:
            return f'Колонки {column_id} не существует, проверьте правильность id'

        sql = f"UPDATE Columns SET {thing_to_change}=? WHERE id=?"
        self.execute(sql, parameters=(new_data, column_id), commit=True)
        return

    def get_last_sequence_number_by_desk_id(self, desk_id):
        sql = "SELECT MAX(sequence_number) FROM Columns WHERE desk_id=?"
        result = self.execute(sql, (desk_id,), fetchone=True)

        return result[0]

    def del_column_by_column_id(self, column_id):
        zxc = self.select_column(id=column_id)
        if zxc:
            self.execute("DELETE FROM Columns WHERE id=?", (column_id,), commit=True)
            self.execute("DELETE FROM Cards WHERE column_id=?", (column_id,), commit=True)
            return True
        else:
            return False

    def change_column_sequence_number(self, column_id, new_sequence_number):
        sql = 'SELECT sequence_number FROM Columns WHERE id=?'
        column_sequence_number = self.select_column(id=column_id)[-1]
        column_desk_id = self.select_column(id=column_id)[1]

        if column_sequence_number < new_sequence_number:
            sql = 'UPDATE Columns SET sequence_number=sequence_number-1 WHERE sequence_number > ? and sequence_number <= ? and desk_id=?'
            self.execute(sql, (column_sequence_number, new_sequence_number, column_desk_id,), commit=True)
            self.update_any_info_about_column(column_id, 'sequence_number', new_sequence_number)
            return True

        elif column_sequence_number > new_sequence_number:
            sql = 'UPDATE Columns SET sequence_number=sequence_number+1 WHERE sequence_number >= ? and sequence_number < ? and desk_id=?'
            self.execute(sql, (new_sequence_number, column_sequence_number, column_desk_id), commit=True)
            self.update_any_info_about_column(column_id, 'sequence_number', new_sequence_number)
            return True

        else:
            return True


#!/usr/bin/env pnpython4

#
# Dump tables in ph5 file to kef format.
#
# Steve Azevedo, April 2007
#

import argparse
import string
import sys
import logging
import time
# This provides the base functionality
from ph5.core import experiment

# Timeseries are stored as numpy arrays

PROG_VERSION = '2019.036'
LOGGER = logging.getLogger(__name__)

#
# These are to hold different parts of the meta-data
#

def init_local():
    global EXPERIMENT_T, EVENT_T, OFFSET_T, SORT_T, RESPONSE_T, REPORT_T
    global ARRAY_T, DAS_T
    global RECEIVER_T, SOH_A, INDEX_T, M_INDEX_T, DASS, TIME_T, TABLE_KEY
    # /Experiment_g/Experiment_t
    EXPERIMENT_T = None
    # /Experiment_g/Sorts_g/Event_t
    EVENT_T = {}
    # /Experiment_g/Sorts_g/Offset_t
    OFFSET_T = {}
    # /Experiment_g/Sorts_g/Sort_t
    SORT_T = None
    # /Experiment_g/Responses_g/Response_t
    RESPONSE_T = None
    # /Experiment_g/Reports_g/Report_t
    REPORT_T = None
    # /Experiment_g/Sorts_g/Array_t_[nnn]
    ARRAY_T = {}
    # /Experiment_g/Receivers_g/Das_g_[sn]/Das_t (keyed on DAS)
    DAS_T = {}
    # /Experiment_g/Receivers_g/Receiver_t
    RECEIVER_T = None
    # /Experiment_g/Receivers_g/Das_g_[sn]/SOH_a_[n] (keyed on DAS then by
    # SOH_a_[n] name)
    SOH_A = {}
    # /Experiment_g/Receivers_g/Index_t
    INDEX_T = None
    # /Experiment_g/Maps_g/Index_t
    M_INDEX_T = None
    # A list of Das_Groups that refers to Das_g_[sn]'s
    DASS = []
    # /Experiment_g/Receivers_g/Time_t
    TIME_T = None
    #
    TABLE_KEY = None

#
# To hold table rows and keys
#

class Rows_Keys(object):
    __slots__ = ('rows', 'keys')

    def __init__(self, rows=None, keys=None):
        self.rows = rows
        self.keys = keys

    def set(self, rows=None, keys=None):
        if rows is not None:
            self.rows = rows
        if keys is not None:
            self.keys = keys

#
# To hold DAS sn and references to Das_g_[sn]
#

class Das_Groups(object):
    __slots__ = ('das', 'node')

    def __init__(self, das=None, node=None):
        self.das = das
        self.node = node

#
# Read Command line arguments
#

def get_args():
    global PH5, PATH, DEBUG, EXPERIMENT_TABLE, SORT_TABLE, OFFSET_TABLE, \
        EVENT_TABLE, ARRAY_TABLE, RESPONSE_TABLE, REPORT_TABLE, \
        RECEIVER_TABLE, DAS_TABLE, TIME_TABLE, TABLE_KEY, INDEX_TABLE, \
        M_INDEX_TABLE, ALL_ARRAYS, ALL_EVENTS, OFILE

    parser = argparse.ArgumentParser(
                                formatter_class=argparse.RawTextHelpFormatter)

    parser.usage = "Version: {0}\ntabletokef\
     --nickname ph5-file-prefix options".format(
        PROG_VERSION)

    parser.description = "Dump a table to a kef file."

    parser.add_argument("-n", "--nickname", dest="ph5_file_prefix",
                        help="The ph5 file prefix (experiment nickname).",
                        metavar="ph5_file_prefix", required=True)

    parser.add_argument("-p", "--path", dest="ph5_path",
                        help=("Path to ph5 files. Default to current "
                              "directory."), default=".",
                        metavar="ph5_path")

    parser.add_argument("-u", "--update_key", dest="update_key",
                        help="Set generated kef file to do an Update on key.",
                        metavar="update_key", type=str)

    parser.add_argument("-d", "--debug", dest="debug", action="store_true",
                        default=False)

    parser.add_argument("-E", "--Experiment_t", dest="experiment_t",
                        action="store_true",
                        default=False,
                        help="Dump /Experiment_g/Experiment_t to a kef file.")

    parser.add_argument("-S", "--Sort_t", dest="sort_t", action="store_true",
                        default=False,
                        help=("Dump /Experiment_g/Sorts_g/Sort_t to a kef "
                              "file."))

    parser.add_argument("-O", "--Offset_t", dest="offset_t_", metavar="a_e",
                        help=("Dump "
                              "/Experiment_g/Sort_g/Offset_t_[arrayID_eventID]"
                              " to a kef file."))

    parser.add_argument("-V", "--Event_t_", dest="event_t_", metavar="n",
                        type=int,
                        help=("Dump /Experiment_g/Sorts_g/Event_t_[n]"
                              "to a kef file."))

    parser.add_argument("--all_events", dest='all_events', action='store_true',
                        default=False,
                        help=("Dump all /Experiment_g/Sorts_g/Event_t_xxx "
                              "to a kef file."))

    parser.add_argument("-A", "--Array_t_", dest="array_t_", metavar="n",
                        type=int,
                        help=("Dump /Experiment_g/Sorts_g/Array_t_[n] "
                              "to a kef file."))

    parser.add_argument("--all_arrays", dest='all_arrays', action='store_true',
                        default=False,
                        help=("Dump all /Experiment_g/Sorts_g/Array_t_xxx "
                              "to a kef file."))

    parser.add_argument("-R", "--Response_t", dest="response_t",
                        action="store_true",
                        default=False,
                        help=("Dump /Experiment_g/Responses_g/Response_t "
                              "to a kef file."))

    parser.add_argument("-P", "--Report_t", dest="report_t",
                        action="store_true",
                        default=False,
                        help=("Dump /Experiment_g/Reports_g/Report_t "
                              "to a kef file."))

    parser.add_argument("-C", "--Receiver_t", dest="receiver_t",
                        action="store_true",
                        default=False,
                        help=("Dump /Experiment_g/Receivers_g/Receiver_t "
                              "to a kef file."))

    parser.add_argument("-I", "--Index_t", dest="index_t", action="store_true",
                        default=False,
                        help=("Dump /Experiment_g/Receivers_g/Index_t "
                              "to a kef file."))

    parser.add_argument("-M", "--M_Index_t", dest="m_index_t",
                        action="store_true",
                        default=False,
                        help=("Dump /Experiment_g/Maps_g/Index_t to a "
                              "kef file."))

    parser.add_argument("-D", "--Das_t", dest="das_t_", metavar="das",
                        help=("Dump /Experiment_g/Receivers_g/Das_g_[das]/"
                              "Das_t to a kef file."))

    parser.add_argument("-T", "--Time_t", dest="time_t", action="store_true",
                        default=False,
                        help=("Dump /Experiment_g/Receivers_g/Time_t "
                              "to a kef file."))

    parser.add_argument("-o", "--outfile", dest="output_file",
                        help="The kef output file to be saved at.",
                        metavar="output_file", default=None)

    args = parser.parse_args()

    PH5 = args.ph5_file_prefix
    PATH = args.ph5_path
    DEBUG = args.debug
    EXPERIMENT_TABLE = args.experiment_t
    SORT_TABLE = args.sort_t
    if args.offset_t_ is not None:
        try:
            OFFSET_TABLE = map(int, args.offset_t_.split("_"))
        except Exception as e:
            LOGGER.error(
                "Offset table should be entered as arrayID underscore "
                "shotLineID, eg. 1_2 or 0_0.")
            LOGGER.error(e.message)
            sys.exit()
    else:
        OFFSET_TABLE = None
    EVENT_TABLE = args.event_t_
    TIME_TABLE = args.time_t
    INDEX_TABLE = args.index_t
    M_INDEX_TABLE = args.m_index_t
    TABLE_KEY = args.update_key
    ARRAY_TABLE = args.array_t_
    ALL_ARRAYS = args.all_arrays
    ALL_EVENTS = args.all_events
    RESPONSE_TABLE = args.response_t
    REPORT_TABLE = args.report_t
    RECEIVER_TABLE = args.receiver_t
    DAS_TABLE = args.das_t_

    table_list = [EXPERIMENT_TABLE, SORT_TABLE, OFFSET_TABLE, EVENT_TABLE,
                  TIME_TABLE, INDEX_TABLE, M_INDEX_TABLE, ARRAY_TABLE,
                  ALL_ARRAYS, ALL_EVENTS, RESPONSE_TABLE, REPORT_TABLE,
                  RECEIVER_TABLE, DAS_TABLE]
    if all(not t for t in table_list):
        LOGGER.error("No table specified for output. See --help for more "
                     "details.")

    # define OFILE to write output
    o_filename = args.output_file
    if o_filename is None:
        OFILE = None
    else:
        OFILE = open(o_filename, 'w')

#
# Initialize ph5 file
#

def initialize_ph5(editmode=False):
    '''   Initialize the ph5 file   '''
    global EX, PATH, PH5

    EX = experiment.ExperimentGroup(PATH, PH5)
    EX.ph5open(editmode)
    EX.initgroup()

#
# Print out report
#
def print_report(text):
    global OFILE
    if OFILE is None:
        print(text)
    else:
        OFILE.write(text + '\n')

#
# Print Rows_Keys
#
def table_print(t, a, fh=None):
    global TABLE_KEY
    global PATH
    global EX
    i = 0
    s = ''
    s = s + \
        "#\n#\t%s\tph5 version: %s\n#\n" % (
            time.ctime(time.time()), EX.version())
    # Loop through table rows
    for r in a.rows:
        i += 1

        s = s + "#   Table row %d\n" % i
        # Print table name
        if TABLE_KEY in a.keys:
            s = s + "{0}:Update:{1} \n".format(t, TABLE_KEY)
        else:
            s = s + t + "\n"
        # Loop through each row column and print
        for k in a.keys:
            s = s + "\t" + str(k) + "=" + str(r[k]) + "\n"
        if fh is None:
            print_report(s)
            s = ''
        else:
            fh.write(s)
            s = ''

def read_time_table():
    global EX, TIME_T

    times, time_keys = EX.ph5_g_receivers.read_time()

    TIME_T = Rows_Keys(times, time_keys)

def read_report_table():
    global EX, REPORT_T

    reports, report_keys = EX.ph5_g_reports.read_reports()

    rowskeys = Rows_Keys(reports, report_keys)

    REPORT_T = rowskeys

def read_experiment_table():
    '''   Read /Experiment_g/Experiment_t   '''
    global EX, EXPERIMENT_T

    exp, exp_keys = EX.read_experiment()

    rowskeys = Rows_Keys(exp, exp_keys)

    EXPERIMENT_T = rowskeys

def read_event_table():
    '''   Read /Experiment_g/Sorts_g/Event_t   '''
    global EX, EVENT_T

    if EVENT_TABLE == 0:
        T = "Event_t"
    else:
        T = "Event_t_{0:03d}".format(EVENT_TABLE)

    try:
        events, event_keys = EX.ph5_g_sorts.read_events(T)
    except Exception:
        LOGGER.error("Can't read {0}.\nDoes it exist?\n".format(T))
        sys.exit()

    rowskeys = Rows_Keys(events, event_keys)

    EVENT_T[T] = rowskeys

def read_all_event_table():
    global EX, EVENT_T
    import re
    EVENT_T_NAME_RE = re.compile("Event_t.*")

    names = EX.ph5_g_sorts.namesRE(EVENT_T_NAME_RE)
    for name in names:
        try:
            events, event_keys = EX.ph5_g_sorts.read_events(name)
        except Exception:
            LOGGER.error("Can't read {0}. Does it exist?".format(name))
            continue

        rowskeys = Rows_Keys(events, event_keys)
        EVENT_T[name] = rowskeys

def read_offset_table():
    '''   Read /Experinent_t/Sorts_g/Offset_t   '''
    global EX, OFFSET_T

    if OFFSET_TABLE[0] == 0 or OFFSET_TABLE[1] == 0:
        name = "Offset_t"
    else:
        name = "Offset_t_{0:03d}_{1:03d}".format(
            OFFSET_TABLE[0], OFFSET_TABLE[1])

    try:
        rows, keys = EX.ph5_g_sorts.read_offset(name)
    except Exception:
        return

    OFFSET_T[name] = Rows_Keys(rows, keys)

def read_sort_table():
    '''   Read /Experiment_t/Sorts_g/Sort_g   '''
    global EX, SORT_T

    sorts, sorts_keys = EX.ph5_g_sorts.read_sorts()

    rowskeys = Rows_Keys(sorts, sorts_keys)

    SORT_T = rowskeys

def read_sort_arrays():
    '''   Read /Experiment_t/Sorts_g/Array_t_[n]   '''
    global EX, ARRAY_T

    # We get a list of Array_t_[n] names here...
    # (these are also in Sort_t)
    names = EX.ph5_g_sorts.names()
    for n in names:
        arrays, array_keys = EX.ph5_g_sorts.read_arrays(n)

        rowskeys = Rows_Keys(arrays, array_keys)
        # We key this on the name since there can be multiple arrays
        ARRAY_T[n] = rowskeys

def read_response_table():
    '''   Read /Experiment_g/Respones_g/Response_t   '''
    global EX, RESPONSE_T

    response, response_keys = EX.ph5_g_responses.read_responses()

    rowskeys = Rows_Keys(response, response_keys)

    RESPONSE_T = rowskeys

def read_receiver_table():
    global EX, RECEIVER_T

    # Read /Experiment_g/Receivers_g/Receiver_t
    receiver, receiver_keys = EX.ph5_g_receivers.read_receiver()
    rowskeys = Rows_Keys(receiver, receiver_keys)
    RECEIVER_T = rowskeys

def read_index_table():
    global EX, INDEX_T

    rows, keys = EX.ph5_g_receivers.read_index()
    INDEX_T = Rows_Keys(rows, keys)

def read_m_index_table():
    global EX, M_INDEX_T

    rows, keys = EX.ph5_g_maps.read_index()
    M_INDEX_T = Rows_Keys(rows, keys)

def read_receivers(das=None):
    '''   Read tables and arrays (except wiggles) in Das_g_[sn]   '''
    global EX, DAS_T, RECEIVER_T, DASS, SOH_A

    dasGroups = EX.ph5_g_receivers.alldas_g()
    if das is None:
        # Get references for all das groups keyed on das
        dass = sorted(dasGroups.keys())
        # Sort by das sn
    else:
        dass = [das]

    for d in dass:
        # Get node reference
        if "Das_g_" + d not in dasGroups:
            continue

        g = dasGroups["Das_g_" + d]
        dg = Das_Groups(d, g)
        # Save a master list for later
        DASS.append(dg)

        # Set the current das group
        EX.ph5_g_receivers.setcurrent(g)

        # Read /Experiment_g/Receivers_g/Das_g_[sn]/Das_t
        das, das_keys = EX.ph5_g_receivers.read_das()
        rowskeys = Rows_Keys(das, das_keys)
        DAS_T[d] = rowskeys

        # Read SOH file(s) for this das
        SOH_A[d] = EX.ph5_g_receivers.read_soh()

#####################################################
# def readPH5
# author: Lan Dam
# updated: 201802
# read data from exp(PH5) to use for KefUtility => KefEdit.py

def readPH5(exp, filename, path, tableType, arg=None):
    # print "readPH5"
    global EX, OFFSET_TABLE, EVENT_TABLE, ARRAY_TABLE, OFFSET_T, EVENT_T,\
        ARRAY_T
    global DAS_T, DASS, SOH_A
    init_local()  # innitiate values and clear cache

    EX = exp

    if tableType == "Experiment_t":
        read_experiment_table()
        return EXPERIMENT_T

    if tableType == "Sort_t":
        read_sort_table()
        return SORT_T

    if tableType == "Offset_t":
        if arg == "Offset_t":
            OFFSET_TABLE = [0]
        else:
            OFFSET_TABLE = map(int, arg.split("_"))
        # read_offset_table() will read from global var. OFFSET_TABLE to add
        # new item into dict. OFFSET_T
        read_offset_table()
        return OFFSET_T

    if tableType == "All_Offset_t":
        for o in EX.Offset_t_names:
            if o == "Offset_t":
                OFFSET_TABLE = [0]
                read_offset_table()
                break
            OFFSET_TABLE = map(int, o.replace("Offset_t_", "").split("_"))
            read_offset_table()
        return OFFSET_T

    if tableType == "Event_t":
        EVENT_TABLE = int(arg)

        # read_event_table() will read from global var. EVENT_TABLE to add new
        # item into dict. EVENT_T
        read_event_table()
        return EVENT_T

    if tableType == "All_Event_t":

        for n in EX.Event_t_names:
            if n == 'Event_t':
                EVENT_TABLE = 0
            else:
                EVENT_TABLE = int(n.replace('Event_t_', ''))
            read_event_table()

        return EVENT_T

    if tableType == "Index_t":
        read_index_table()
        return INDEX_T

    if tableType == "Map_Index_t":
        read_m_index_table()
        return M_INDEX_T

    if tableType == "Time_t":
        read_time_table()
        return TIME_T

    if tableType == "Array_t":
        ARRAY_TABLE = arg
        read_sort_table()
        read_sort_arrays()
        arrays = ARRAY_T.keys()
        for a in arrays:
            n = int(string.split(a, '_')[2])
            if n == int(ARRAY_TABLE):
                return ARRAY_T[a]

    if tableType == "All_Array_t":
        read_sort_table()
        read_sort_arrays()
        arrays = ARRAY_T.keys()
        return ARRAY_T

    if tableType == "Response_t":
        read_response_table()
        return RESPONSE_T

    if tableType == "Report_t":
        read_report_table()
        return REPORT_T

    if tableType == "Receiver_t":
        read_receiver_table()
        return RECEIVER_T

    if tableType == "Das_t":
        read_receivers(arg)
        return DAS_T

def main():
    global PH5, PATH, DEBUG, EXPERIMENT_TABLE, SORT_TABLE, OFFSET_TABLE,\
        EVENT_TABLE, \
        ARRAY_TABLE, RESPONSE_TABLE, REPORT_TABLE, RECEIVER_TABLE, DAS_TABLE,\
        TIME_TABLE, INDEX_TABLE

    init_local()

    get_args()

    initialize_ph5()

    if EXPERIMENT_TABLE:
        read_experiment_table()
        table_print("/Experiment_g/Experiment_t", EXPERIMENT_T)

    if SORT_TABLE:
        read_sort_table()
        table_print("/Experiment_g/Sorts_g/Sort_t", SORT_T)

    if OFFSET_TABLE:
        read_offset_table()
        keys = OFFSET_T.keys()
        for k in keys:
            table_print("/Experiment_g/Sorts_g/{0}".format(k), OFFSET_T[k])

    if EVENT_TABLE is not None:
        read_event_table()
        keys = EVENT_T.keys()
        for k in keys:
            table_print("/Experiment_g/Sorts_g/{0}".format(k), EVENT_T[k])
    elif ALL_EVENTS is not False:
        read_all_event_table()
        keys = EVENT_T.keys()
        for k in keys:
            table_print("/Experiment_g/Sorts_g/{0}".format(k), EVENT_T[k])

    if INDEX_TABLE:
        read_index_table()
        table_print("/Experiment_g/Receivers_g/Index_t", INDEX_T)

    if M_INDEX_TABLE:
        read_m_index_table()
        table_print("/Experiment_g/Maps_g/Index_t", M_INDEX_T)

    if TIME_TABLE:
        read_time_table()
        table_print("/Experiment_g/Receivers_g/Time_t", TIME_T)

    if ARRAY_TABLE:
        if not SORT_T:
            read_sort_table()

        read_sort_arrays()
        arrays = ARRAY_T.keys()
        for a in arrays:
            n = int(string.split(a, '_')[2])
            if n == int(ARRAY_TABLE):
                table_print("/Experiment_g/Sorts_g/" + a, ARRAY_T[a])
    elif ALL_ARRAYS:
        if not SORT_T:
            read_sort_table()

        read_sort_arrays()
        arrays = ARRAY_T.keys()
        for a in arrays:
            table_print("/Experiment_g/Sorts_g/" + a, ARRAY_T[a])

    if RESPONSE_TABLE:
        read_response_table()
        table_print("/Experiment_g/Responses_g/Response_t", RESPONSE_T)

    if REPORT_TABLE:
        read_report_table()
        table_print("/Experiment_g/Reports_g/Report_t", REPORT_T)

    if RECEIVER_TABLE:
        read_receiver_table()
        table_print("/Experiment_g/Receivers_g/Receiver_t", RECEIVER_T)

    if DAS_TABLE:
        read_receivers(DAS_TABLE)
        dass = DAS_T.keys()
        for d in dass:
            table_print("/Experiment_g/Receivers_g/Das_g_" +
                        d + "/Das_t", DAS_T[d])

    EX.ph5close()
    if OFILE is not None:
        OFILE.close()

if __name__ == '__main__':
    main()


import httplib
import socket
import urlparse

import eventlet
import httplib2
from oslo.config import cfg
import webob

from quantum.agent.linux import daemon
from quantum.common import config
from quantum.common import utils
from quantum.openstack.common import log as logging
from quantum import wsgi

proxy_socket = cfg.StrOpt('metadata_proxy_socket',
                          default='$state_path/metadata_proxy',
                          help=_('Location of Metadata Proxy UNIX domain '
                                 'socket'))

cfg.CONF.register_opt(proxy_socket)

LOG = logging.getLogger(__name__)

class UnixDomainHTTPConnection(httplib.HTTPConnection):
    """Connection class for HTTP over UNIX domain socket."""
    def __init__(self, host, port=None, strict=None, timeout=None,
                 proxy_info=None):
        httplib.HTTPConnection.__init__(self, host, port, strict)
        self.timeout = timeout

    def connect(self):
        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        if self.timeout:
            self.sock.settimeout(self.timeout)
        self.sock.connect(cfg.CONF.metadata_proxy_socket)

class NetworkMetadataProxyHandler(object):
    """Proxy AF_INET metadata request through Unix Domain socket.

       The Unix domain socket allows the proxy access resource that are not
       accessible within the isolated tenant context.
    """

    def __init__(self, network_id=None, router_id=None):
        self.network_id = network_id
        self.router_id = router_id

        if network_id is None and router_id is None:
            msg = _('network_id and router_id are None. One must be provided.')
            raise ValueError(msg)

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        LOG.debug(_("Request: %s"), req)
        try:
            return self._proxy_request(req.remote_addr,
                                       req.method,
                                       req.path_info,
                                       req.query_string,
                                       req.body)
        except Exception:
            LOG.exception(_("Unexpected error."))
            msg = _('An unknown error has occurred. '
                    'Please try your request again.')
            return webob.exc.HTTPInternalServerError(explanation=unicode(msg))

    def _proxy_request(self, remote_address, method, path_info,
                       query_string, body):
        headers = {
            'X-Forwarded-For': remote_address,
        }

        if self.router_id:
            headers['X-Quantum-Router-ID'] = self.router_id
        else:
            headers['X-Quantum-Network-ID'] = self.network_id

        url = urlparse.urlunsplit((
            'http',
            '169.254.169.254',  # a dummy value to make the request proper
            path_info,
            query_string,
            ''))

        h = httplib2.Http()
        resp, content = h.request(
            url,
            method=method,
            headers=headers,
            body=body,
            connection_type=UnixDomainHTTPConnection)

        if resp.status == 200:
            LOG.debug(resp)
            LOG.debug(content)
            return content
        elif resp.status == 404:
            return webob.exc.HTTPNotFound()
        elif resp.status == 409:
            return webob.exc.HTTPConflict()
        elif resp.status == 500:
            msg = _(
                'Remote metadata server experienced an internal server error.'
            )
            LOG.debug(msg)
            return webob.exc.HTTPInternalServerError(explanation=unicode(msg))
        else:
            raise Exception(_('Unexpected response code: %s') % resp.status)

class ProxyDaemon(daemon.Daemon):
    def __init__(self, pidfile, port, network_id=None, router_id=None):
        super(ProxyDaemon, self).__init__(pidfile)
        self.network_id = network_id
        self.router_id = router_id
        self.port = port

    def run(self):
        handler = NetworkMetadataProxyHandler(
            self.network_id,
            self.router_id)
        proxy = wsgi.Server('quantum-network-metadata-proxy')
        proxy.start(handler, self.port)
        proxy.wait()

def main():
    eventlet.monkey_patch()
    opts = [
        cfg.StrOpt('network_id'),
        cfg.StrOpt('router_id'),
        cfg.StrOpt('pid_file'),
        cfg.BoolOpt('daemonize', default=True),
        cfg.IntOpt('metadata_port',
                   default=9697,
                   help=_("TCP Port to listen for metadata server "
                          "requests.")),
    ]

    cfg.CONF.register_cli_opts(opts)
    # Don't get the default configuration file
    cfg.CONF(project='quantum', default_config_files=[])
    config.setup_logging(cfg.CONF)
    utils.log_opt_values(LOG)
    proxy = ProxyDaemon(cfg.CONF.pid_file,
                        cfg.CONF.metadata_port,
                        network_id=cfg.CONF.network_id,
                        router_id=cfg.CONF.router_id)

    if cfg.CONF.daemonize:
        proxy.start()
    else:
        proxy.run()


"""Import packages and libraries"""
import cv2 as cv
import numpy as np
from tensorflow import keras

# Parameters: (Choose which classification model to use, Need to run model first before this)
mod = "EfficientNetV2"  # EfficientNetV2/InceptionNetV2/CNN

if mod == "EfficientNetV2":
    file = "./effnet_weights.h5"
elif mod == "InceptionNetV2":
    file = "./inception_weights.h5"
else:
    file = "./cnn_weights.h5"

# Cascade Classifier for face detection
classifier = cv.CascadeClassifier('haarcascade_frontalface_default.xml')
labels_dict = {0: 'without_mask', 1: 'with_mask'}
size = 4
# load model for classification
load_model = keras.models.load_model(file)
cv.namedWindow("Face Mask Detection")
cam = cv.VideoCapture(0)

# Two different colors to differ wearing a mask or not
color_dict = {0: (0, 0, 255), 1: (0, 255, 0)}

# sr = cv.dnn_superres.DnnSuperResImpl_create()
# path = "models/LapSRN_x8.pb"
# sr.readModel(path)
# sr.setModel("lapsrn", 8)

while True:
    rval, image = cam.read()
    image = cv.flip(image, 1, 1)
    small = cv.resize(image, (image.shape[1] // size, image.shape[0] // size))
    faces = classifier.detectMultiScale(small)
    for f in faces:
        # Find the bounding box of the face
        (x, y, w, h) = [v * size for v in f]
        face_img = image[y:y + h, x:x + w]
        # face_img = sr.upsample(face_img)
        # Preprocess the image for prediction
        resized = cv.resize(face_img, (224, 224))
        reshaped = np.reshape(resized, (1, 224, 224, 3))
        reshaped = np.vstack([reshaped])
        # predict the image given
        result = load_model.predict(reshaped)
        result = np.array(result)
        result = result.reshape((1))
        result = result.clip(0, 1)
        label = ""
        # based on the results given, determine if wearing a mask or not
        if result[0] < 0.5:
            label = 'without_mask'
            color = (0, 0, 255)
        else:
            label = 'with_mask'
            color = (0, 255, 0)
        # Draw rectangle around the face and color it with the classified color
        cv.rectangle(image, (x, y), (x + w, y + h), color, 2)
        cv.rectangle(image, (x, y - 40), (x + w, y), color, -1)
        cv.putText(image, label, (x, y - 10), cv.FONT_HERSHEY_SIMPLEX, 0.8,
                   (255, 255, 255), 2)

    if image is not None:
        cv.imshow('Face Mask Detection', image)
    # to exit the camera feed
    key = cv.waitKey(10)
    if key == 27:
        break
cam.release()
cv.destroyAllWindows()


# this is a script that runs the experiments of subset attack on
# the scheme for categorical data based on a neighbourhood search
# approach
import sys
sys.path.append("/home/sarcevic/fingerprinting-toolbox/")

import random
from datetime import datetime
import numpy as np

from attacks.combined_attack import CombinedAttack
from scheme import CategoricalNeighbourhood
import itertools

n_experiments = 1 #6 #10  # number of times we attack the same fingerprinted file
n_fp_experiments = 1 #15 #25  # number of times we run fp insertion

size_of_subset = np.array([0.9, 0.95, 1])
fractions = np.array([0.9, 0.95, 1])
columns = np.array([1, 2])
a = [size_of_subset, columns, fractions]
combinations = list(itertools.product(*a))
results = []
gamma = 3; xi = 2; fingerprint_bit_length = 8

scheme = CategoricalNeighbourhood(gamma=gamma, xi=xi, fingerprint_bit_length=fingerprint_bit_length)
attack = CombinedAttack()
data = 'german_credit'

f = open("robustness_analysis/categorical_neighbourhood/log/combined_attack_" + data + ".txt", "a+")

for combo in combinations:
    # for reproducibility
    seed = 332
    random.seed(seed)

    correct, misdiagnosis = 0, 0
    for i in range(n_fp_experiments):
        # fingerprint the data
        secret_key = random.randint(0, 1000)
        fp_dataset = scheme.insertion(dataset_name=data, buyer_id=1, secret_key=secret_key)

        for j in range(n_experiments):
            # perform the attack
            release_data = attack.run(dataset=fp_dataset, fraction_subset=combo[0], number_of_columns=combo[1],
                                      fraction_flipping=combo[2])
            # try to extract the fingerprint
            suspect = scheme.detection(dataset_name=data, real_buyer_id=1, secret_key=secret_key,
                                dataset=release_data)
            if suspect == 1:
                correct += 1
            elif suspect != -1:
                    misdiagnosis += 1

    print("\n\n--------------------------------------------------------------\n\n")
    print("Data: " + data)
    print("(size of subset, columns, flipped, gamma, xi, length of a fingerprint): " +
          str((combo[0], combo[1], combo[2], gamma, xi, fingerprint_bit_length)))
    print("Correct: " + str(correct) + "/" + str(n_experiments*n_fp_experiments))
    print("Wrong: " + str(n_experiments*n_fp_experiments - correct) + "/" + str(n_experiments*n_fp_experiments)
          + "\n\t- out of which misdiagnosed: " + str(misdiagnosis))

    # write to log file
    f.write(str(datetime.fromtimestamp(int(datetime.timestamp(datetime.now())))))
    f.write("\nseed: " + str(seed))
    f.write("\nData: " + data)
    f.write("\n(size of subset, columns, flipped, gamma, xi, length of a fingerprint): " + str((combo[0], combo[1], combo[2], gamma, xi,
                                                                            fingerprint_bit_length)))
    f.write("\nCorrect: " + str(correct) + "/" + str(n_experiments*n_fp_experiments))
    f.write("\nWrong: " + str(n_experiments*n_fp_experiments - correct) + "/" + str(n_experiments*n_fp_experiments)
          + "\n\t- out of which misdiagnosed: " + str(misdiagnosis))
    f.write("\n\n--------------------------------------------------------------\n\n")

    results.append(correct)

f.write("SUMMARY\n")
f.write(str(datetime.fromtimestamp(int(datetime.timestamp(datetime.now())))))
f.write("\n(gamma, xi, length of a fingerprint): " + str((gamma, xi, fingerprint_bit_length)))
f.write("\n" + str(combinations))
f.write("\nCorrect: " + str(results) + "\n\t/" + str(n_experiments * n_fp_experiments))
f.write("\n\n--------------------------------------------------------------\n\n")
f.close()


import json
import random
import requests
import datetime
import time
import os, sys
import urllib.request
import yaml, pgeocode

def create_directory():
    #create loacation,image,date folder
    l = ['output','tinder',cfg['city'],"images",date]
    # path = cfg['output_folder']
    path = '../../..'
    for i in range(5):
        path = path+'/'+l[i]
        if not os.path.exists(path):
            os.mkdir(path)

def get_self():
    url = cfg['host'] + '/profile'
    try:
        r = requests.get(url, headers=headers)
        return r.json()
    except requests.exceptions.RequestException as e:
        print(e)

def change_preferences(**kwargs):
    '''
    ex: change_preferences(age_filter_min=30, gender=0)
    kwargs: a dictionary - whose keys become separate keyword arguments and the values become values of these arguments
    age_filter_min: 18..46
    age_filter_max: 22..55
    age_filter_min <= age_filter_max - 4
    gender: 0 == seeking males, 1 == seeking females
    distance_filter: 1..100
    discoverable: true | false
    {"photo_optimizer_enabled":false}
    '''
    try:
        url = cfg['host'] + '/profile'
        r = requests.post(url, headers=headers, data=json.dumps(kwargs))
    except requests.exceptions.RequestException as e:
        print(e)

def get_geolocation(cn, pincode):
    nomi = pgeocode.Nominatim(cn)
    res = dict(nomi.query_postal_code(pincode))
    lat = res['latitude']
    lon = res['longitude']
    return lat, lon

def update_location():
    '''
    Updates your location to the given latitude and longitude - cannot call the API multiple times at one go. Give a 20-25 mins gap between calls.
    '''
    lat, lon = get_geolocation(cfg['country_name'], cfg['pincode'])
    try:
        url = cfg['host'] + '/user/ping'
        r = requests.post(url, headers=headers, data=json.dumps({"lat": lat, "lon": lon}))
    except requests.exceptions.RequestException as e:
        print(e)

def get_recommendations():
    '''
    Returns a list of users that you can swipe on
    '''
    url = cfg['host']+'/user/recs'
    try:
        r = requests.get(url, headers=headers)
        return r.json()
    except requests.exceptions.RequestException as e:
        print("Something went wrong with getting recomendations:", e)

def swipe(profiles, rcounter):
    last_swipe_count = 0
    rightswipe = set()
    for i in range(random.randint(0,3)): #level 1 of randomness(0-5 swipes)
        rightswipe.add(random.randint(0,len(profiles)))  #level 2 of randomness(profile no 0-22)

    rightswipe = list(rightswipe)
    for index, user in enumerate(profiles):
        try:
            if index in rightswipe and rcounter < cfg['right_swipes']:
                url = cfg['host'] + '/like/%s' % user['_id']
                r = requests.get(url, headers=headers)
                rcounter+=1
                last_swipe_count +=1
                print(user['_id'], "swiped right")
            else:
                url = cfg['host'] + '/pass/%s' % user['_id']
                r = requests.get(url, headers=headers)
                print(user['_id'], "swiped left")
        except requests.exceptions.RequestException as e:
            print(e)
    return last_swipe_count, rcounter

def extract_images(profiles):
    if cfg['gender_filter']==1:
        gender = "_female"
    else:
        gender = "_male"
    for profile in profiles:
        count = 1
        _id = profile['_id']
        path = image_path+'/'+_id+gender
        if not os.path.exists(path):
            os.mkdir(path);
        for photo in profile['photos']:
            filename = path+'/%d.jpg'%count
            url = photo['processedFiles'][0]['url']
            try:
                urllib.request.urlretrieve(url, filename)
            except:
                pass
            count+=1

def export_results(dump,path):
    with open(path, 'w+') as fp:
        json.dump(dump, fp)

def import_results(path):
    with open(path) as fp:
        return json.load(fp)

def retrieve_lostdata(path, flag):
    print("## BACKFILL INITIATED ##")

    file = cfg['output_folder']+path.split('/')[4]+'/'+path.split('/')[-1]+'.json'
    try:
        temp_dict = import_results(file)
    except json.decoder.JSONDecodeError:
        temp_dict = {
        "name" : cfg['name'],
        "city" : cfg['city'],
        "position" : {},
        "date" : date,
        "last_swipe_count" : 0,
        "results" : []
        }

    if flag is True:
            temp_dict['results'] = []

    _ids = list(map(lambda d: d['_id'], temp_dict['results']))
    for num, folder in enumerate(os.listdir(path)):
        _id = folder.split('_')[0]
        if flag is False and _id in _ids:
            print("ID already present", _id)
            continue

        print(num, "BACKFILL ID: ", _id)
        url = cfg['host'] + '/user/%s' % _id
        rec = requests.get(url, headers=headers)
        if rec.status_code == 401:
            print("TOKEN EXPIRED")
        try:
            if rec.status_code == 200:
                temp_dict['results'].append(rec.json()['results'])
        except KeyboardInterrupt:

            export_results(temp_dict,file)

            break

    export_results(temp_dict,file)
    print("## BACKFILL COMPLETE ##")

def get_extractions():
    # set preferences
    # checking if the script is being run for the first time during the day(file exists or not)
    rcounter = 0
    file_exists = os.path.isfile("%s.json" % op_file_path)
    if not file_exists:
        #get source profile location
        temp_dict = {
        "name" : cfg['name'],
        "city" : cfg['city'],
        "position" : {},
        "date" : date,
        "last_swipe_count" : 0,
        "results" : []

        }
        coordinates = get_self()['pos_info']
        temp_dict['position'] = coordinates
        # export_results(temp_dict,"%s.json" % op_file_path)

    else:
        temp_dict = import_results("%s.json" % op_file_path)
    while(rcounter<cfg['right_swipes'] and temp_dict['last_swipe_count']+rcounter<=90):
        try:
            print("### Image and text extraction started ###")
            rec = get_recommendations()
            print("got", len(rec['results']), "profiles")
            temp_dict['results'] = temp_dict['results'] + rec['results']
            export_results(temp_dict,"%s.json" % op_file_path)
            last_swipe_count, rcounter = swipe(rec['results'], rcounter)
            temp_dict['last_swipe_count']+=last_swipe_count
            extract_images(rec['results'])
        except KeyboardInterrupt:
            break

    export_results(temp_dict,"%s.json" % op_file_path)
    print(len(temp_dict['results']),"profiles done")

if __name__ == '__main__':
    date = str(datetime.date.today())
    with open('../../../config/tinder.yaml', 'r') as ymlfile:
        cfg = yaml.load(ymlfile)

    main_path = cfg['output_folder']+'/tinder/'+cfg['city']
    op_file_path = main_path+'/'+date
    image_path = main_path+"/images/"+date
    headers = {
        'app_version': str(cfg['app_version']),
        'platform': cfg['platform'],
        "content-type": "application/json",
        "User-agent": cfg['User_agent'],
        "X-Auth-Token": cfg['tinder_token'],
    }
    if cfg['backfill']:
        retrieve_lostdata(cfg['backfill_folder'], cfg['backfill_force'])
    else:
        create_directory()
        update_location()
        change_preferences(age_filter_min=cfg['min_age'], age_filter_max=cfg['max_age'],  gender_filter = cfg['gender_filter'], gender = cfg['gender'])
        get_extractions()


import sys
sys.setrecursionlimit(1000000)

c.Spawner.default_url = '/lab'

c.JupyterHub.admin_access = True
c.JupyterHub.authenticator_class = 'nativeauthenticator.NativeAuthenticator'
c.Authenticator.admin_users = {"clamorz"}
c.Authenticator.open_signup = True
c.Authenticator.ask_email_on_signup = True

import pwd
import subprocess

def pre_spawn_hook(spawner):
    username = spawner.user.name
    try:
        pwd.getpwnam(username)
    except KeyError:
        subprocess.check_call(['useradd', '-ms', '/bin/bash', username])

c.Spawner.pre_spawn_hook = pre_spawn_hook

# import shutil
# c.LanguageServerManager.language_servers = {
#     "C": {
#         "argv": [shutil.which("clangd-10")],
#         "languages": ["c", "C", "c++", "C++17", "xcpp17", "c++src", "text/x-c++src"],
#         "version": 1
#     }
# }


import requests
from abc import ABC, abstractmethod
from employer import Employer
from vacancy import Vacancy

class JobAPI(ABC):
    """
    Абстрактный класс для работы с API сайтов с вакансиями
    """

    @abstractmethod
    def get_employers(self, keyword) -> dict:
        """
        Метод для получения информации о работодателях по API
        keyword: Ключевое слово запроса
        """
        pass

    @abstractmethod
    def get_employer_vacancies(self, employer_id) -> dict:
        """
        Метод для получения вакансий работодателя по API
        keyword: Ключевое слово запроса
        """
        pass

class HeadHunterAPI(JobAPI):
    """
    Класс для получений вакансий с HeadHunter (hh.ru)
    """
    def __init__(self, page=0, per_page=50):
        self.__page = page  # номер страницы
        self.__per_page = per_page  # количество получаемых вакансий на одну страницу

    @property
    def page(self):
        return self.__page

    @page.setter
    def page(self, page: int):
        self.__page = page

    @property
    def per_page(self):
        return self.__per_page

    @per_page.setter
    def per_page(self, per_page: int):
        self.__per_page = per_page

    def get_employers(self, keyword) -> list:
        """
        Метод для получения информации о работодателях с hh.ru по API
        keyword: Ключевое слово запроса
        """
        employers = []  # список работодателей

        # параметры запроса по API
        employers_parameters = {
            "text": keyword,
            "only_with_vacancies": True,
            "per_page": self.__per_page,
            "page": self.__page
        }

        # получение работодателей по API
        hh_employers = requests.get(url='https://api.hh.ru/employers', params=employers_parameters).json()

        for hh_employer in hh_employers["items"]:
            employer = Employer(employer_id=hh_employer["id"],
                                employer_name=hh_employer["name"],
                                url=hh_employer["alternate_url"],
                                url_vacancies=hh_employer["vacancies_url"],
                                open_vacancies_numbers=hh_employer["open_vacancies"])
            employers.append(employer)

        return employers

    def get_employer_vacancies(self, employer_id) -> list:
        """
        Метод для получения вакансий работодателя с hh.ru по API
        keyword: Ключевое слово запроса
        """
        vacancies = []  # список вакансий работодателя

        # параметры запроса по API
        parameters = {
            "per_page": self.__per_page,
            "page": self.__page
        }

        # получение вакансий работодателя по API
        hh_employer_vacancies = requests.get(url=f'https://api.hh.ru/vacancies?employer_id={employer_id}',
                                             params=parameters).json()

        for vacancy in hh_employer_vacancies["items"]:
            if vacancy["salary"] is not None and vacancy["salary"]["from"] is not None:
                salary_from = vacancy["salary"]["from"]
            else:
                salary_from = 0

            if vacancy["salary"] is not None and vacancy["salary"]["to"] is not None:
                salary_to = vacancy["salary"]["to"]
            else:
                salary_to = 0

            if vacancy["address"] is not None and vacancy["address"]["city"] is not None:
                city = vacancy["address"]["city"]
            else:
                city = ""

            if vacancy["salary"] is not None and vacancy["salary"]["currency"] is not None:
                currency = vacancy["salary"]["currency"]
            else:
                currency = ""

            vacancy = Vacancy(vacancy_id=vacancy["id"],
                              title=vacancy["name"],
                              url=vacancy["alternate_url"],
                              salary_from=salary_from,
                              salary_to=salary_to,
                              description=f'{vacancy["snippet"]["requirement"]}\n'
                                          f'{vacancy["snippet"]["responsibility"]}',
                              city=city,
                              currency=currency,
                              employer_id=vacancy["employer"]["id"])

            vacancies.append(vacancy)

        return vacancies


import argparse
import getpass
import os
import random
from datetime import datetime
from typing import List

from dt_shell import DTCommandAbs, DTShell, UserError
from dt_shell.env_checks import check_docker_environment
from duckietown_docker_utils import ENV_REGISTRY

class DTCommand(DTCommandAbs):
    @staticmethod
    def command(shell: DTShell, args: List[str]):
        from duckietown_docker_utils import generic_docker_run

        parser = argparse.ArgumentParser()

        parser.add_argument(
            "--image",
            default="${%s}/duckietown/duckietown-challenges-cli:daffy" % ENV_REGISTRY,
            help="Which image to use",
        )

        # parser.add_argument('--entrypoint', default=None)
        parser.add_argument("--shell", default=False, action="store_true")
        parser.add_argument("--root", default=False, action="store_true")
        # parser.add_argument('--dev', default=False, action='store_true')
        parser.add_argument("--no-pull", action="store_true", default=False, help="")
        # parser.add_argument('cmd', nargs='*')
        parsed, rest = parser.parse_known_args(args=args)

        if "DT_MOUNT" in os.environ:
            development = True
        else:
            development = False
        # dtslogger.info(str(dict(args=args, parsed=parsed, rest=rest)))
        try:
            dt1_token = shell.get_dt1_token()
        except:
            dt1_token = None
        # username, secret = get_dockerhub_username_and_password()
        client = check_docker_environment()
        shell_config = shell.shell_config

        timestamp = "{:%Y_%m_%d_%H_%M_%S_%f}".format(datetime.now())
        container_name = f"build_utils_{timestamp}_{random.randint(0, 10)}"
        user = getpass.getuser()
        logname = f"/tmp/{user}/duckietown/dt-shell-commands/build_utils/{container_name}.txt"

        no_pull = parsed.no_pull
        gdr = generic_docker_run(
            client,
            entrypoint="dt-build_utils-cli",
            as_root=parsed.root,
            image=parsed.image,
            commands=rest,
            shell=parsed.shell,
            docker_secret=None,
            docker_username=None,
            dt1_token=dt1_token,
            development=development,
            container_name=container_name,
            pull=not no_pull,
            read_only=False,
            detach=True,
            logname=logname,
            docker_credentials=shell_config.docker_credentials,
        )
        if gdr.retcode:
            msg = f"Execution of docker image failed. Return code: {gdr.retcode}."
            msg += f"\n\nThe log is available at {logname}"
            raise UserError(msg)


from prefect.deployments import Deployment
from prefect.filesystems import GitHub

github_block = GitHub.load("github-block")

github_dep = Deployment.build_from_flow(
    flow=parent_flow,
    name="github-flow",
    storage = github_block
)

if __name__ == "__main__":
    github_dep.apply()


from rest_framework.decorators import api_view
from rest_framework.response import Response
import numpy as np
from sklearn import metrics
from sklearn.cluster import KMeans
import json
import os, sys
from api.models import Student_activity
from api.serializers import StudentActivitySerializer
from users.models import CustomUser
from rest_framework import status
currentdir = os.path.dirname(os.path.realpath(__file__))
parentdir = os.path.dirname(currentdir)
sys.path.append(parentdir)
from plotting.plot_2d import plot_2d
# Create your views here.
@api_view(['GET'])
def index_page(request):
    return_data = {
        "error" : "0",
        "message" : "Successful",
    }
    return Response(return_data)

def kmeans_cluster(X, n_clusters, user_id, features):

    kmeans = KMeans(n_clusters)
    kmeans.fit(X)
    y_kmeans = kmeans.predict(X)
    silhouette_score=metrics.silhouette_score(X, y_kmeans)
    X = np.array(X)
    plt_url = 'media/{}'.format(user_id)
    if not os.path.exists(plt_url):
        os.makedirs(plt_url)
    title = "Kmeans Clustering"
    plt_url += '/{}.png'.format(title.replace(" ", "_"))
    i=0
    while os.path.exists(f"{plt_url}_{i}.png"):
        i += 1
    plt_url = '{}_{}.png'.format(plt_url,i)
    plot_2d(X, y_kmeans, plt_url, title, features)
    return y_kmeans, kmeans.inertia_, silhouette_score, plt_url

@api_view(["POST"])
def get_kmeans(request):
    print(request.user.id)
    user_id=request.user.id
    if(user_id==None):
        user_id=1
    try:
        # load request from json
        data = json.loads(request.body)
        #print("data", data)
        # data contains two fileds: k: number of cluster, train: matrix(size: m*n)
        k = data['k']
        train_data = data['train']
        if ('header' in data):
            header = data['header']
        else:
            header = None
        features = None
        #train_data = request.GET.get('data')
        if k is not None:
            # Datapreprocessing Convert the values to float
            k = int(k)
            #print("k",k)
            # # Filtering the rows which contains None
            train_data = list(filter(any, train_data))
            train_data = [list(filter(None, lst)) for lst in train_data]
            if (not header):
                features = None
            else:
                features = train_data.pop(0)

            train_data = np.asarray(train_data, dtype=np.float64)
            #print(train_data)
            #print("train_data",train_data)
            y_kmeans, ssd_kmeans, silhouette_score, plt_url = kmeans_cluster(train_data, k, user_id, features)

            input_output = np.concatenate((y_kmeans.reshape(-1, 1), train_data), axis=1)

            if(features):
                features.insert(0, " ")
                features = np.array(features)
                input_output = np.concatenate((features.reshape(1, -1), input_output), axis=0)
            result = {
                'error' : '0',
                'message' : 'Successfull',
                'y_kmeans' : y_kmeans.reshape(-1,1),
                'input_output': input_output,
                'ssd' : ssd_kmeans,
                'silhouette_score' : silhouette_score,
                'plt_url' : plt_url
            }
            #print("concate:",np.concatenate((train_data, y_kmeans.T), axis=1))
            user = CustomUser.objects.get(id=user_id)
            activity = Student_activity.objects.create(user=user, ml_model="kmeans", n_rows=train_data.shape[0], n_columns=train_data.shape[1])
            serializer = StudentActivitySerializer(activity)
            #rsp = {'message': 'Activity Created', 'result': serializer.data}
            #print(rsp)
        else:
            response = {'error': '1', 'message': ['The value of K is missing']}
            return Response(response, status=status.HTTP_400_BAD_REQUEST)
    except Exception as e:
        response = {
            'error' : '2',
            "message": [str(e)]
        }
        if (str(e) == "could not convert string to float: '?'"):
            response = {
                'error': '2',
                "message": ["The data should contain only decimal values. Check if any datapoint is missing in the table or the attached file"]
            }
        return Response(response, status=status.HTTP_400_BAD_REQUEST)
    return Response(result)


from pprint import pformat
from io import BytesIO

import core
import settings

PLUGINVERSION = 2
# Always name this variable as `plugin`
# If you dont, module loader will fail to load the plugin!
plugin = core.Plugin()

@plugin.command(command="//msgdump",
                hidden=True)
def dump(bot, update, user, args):
    text = pformat(update.message.to_dict())
    text_obj = BytesIO()
    text_obj.name = "%s.txt" % update.update_id
    text_obj.write(bytes(text, "utf-8"))
    text_obj.seek(0)
    return core.message(file=text_obj)

@plugin.command(command="//delmsg",
                inline_supported=False,
                hidden=True)
def msgdel(bot, update, user, args):
    if user.id == settings.ADMIN:
        update.message.reply_to_message.delete()

@plugin.command(command="//exec",
                hidden=True)
def docode(bot, update, user, args):
    if user.id == settings.ADMIN:
        return core.message(eval(" ".join(args)))

@plugin.command(command="//plugins", hidden=True)
def coreplug_list(bot, *_):
    message = []
    for plugin in bot.modloader.plugins:
        txt = ''
        if plugin.state == core.constants.OK:
            txt += "✅"
        else:
            txt += "⛔"
        txt += plugin.name
        message.append(txt)
    message = sorted(message)
    message.reverse()
    return core.message("\n".join(message))

@plugin.command(command="//sideload", hidden=True)
def coreplug_load(self, bot, update, user, args):
    args = " ".join(args)
    if user.id == settings.ADMIN:
        self.logger.info("Reload requested.")
        update.message.reply_text("Loading " + args)
        self.load_plugin(args)
        return self.coreplug_list()
    else:
        return core.message("Access Denied.")


"""add packs

Revision ID: ec89740b6903
Revises: da7556b25d75
Create Date: 2022-08-02 14:55:41.971432

"""
import sqlalchemy as sa
import sqlmodel
from alembic import op

# revision identifiers, used by Alembic.
revision = "ec89740b6903"
down_revision = "da7556b25d75"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "pack",
        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.PrimaryKeyConstraint("id"),
        sa.UniqueConstraint("name"),
    )
    op.create_index(op.f("ix_pack_name"), "pack", ["name"], unique=False)
    op.create_table(
        "packsentence",
        sa.Column("sentence_id", sa.Integer(), nullable=False),
        sa.Column("pack_id", sa.Integer(), nullable=False),
        sa.ForeignKeyConstraint(
            ["pack_id"],
            ["pack.id"],
        ),
        sa.ForeignKeyConstraint(
            ["sentence_id"],
            ["sentence.id"],
        ),
        sa.PrimaryKeyConstraint("sentence_id", "pack_id"),
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("packsentence")
    op.drop_index(op.f("ix_pack_name"), table_name="pack")
    op.drop_table("pack")
    # ### end Alembic commands ###