import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Requirements_and_value_identification_tool.settings')

import django
from datetime import datetime
django.setup()
from web_app.models import *

def populate():

    sprints = [
        {
            'sprint_id': '0',
            'dashboard_id': '0000',
            'start_date': date.today(),
            'end_date': date.today(),
            'stories': [],
            'story_list': ''
        }
    ]

    for sprint in sprints:
        add_sprint(sprint['sprint_id'], sprint['dashboard_id'], sprint['start_date'], sprint['end_date'], sprint['stories'], sprint['story_list'])

    tracking_columns = [
        {
            'column_id': '0',
            'dashboard_id': '0000',
            'team_id': '0000',
            'title': 'Backlog',
            'mark_as_complete': False,
            'stories': [],
            'WIP': 10
        }
    ]

    for column in tracking_columns:
        add_tracking_column(column['column_id'], column['dashboard_id'], column['team_id'], column['title'], column['mark_as_complete'], column['stories'], column['WIP'])

    tags = [
        {
            'tag_id': '0',
            'team_id': '0000',
            'title': 'Test tag',
            'description': "A tag used for testing",
            'colour': 'ffffff'
        }
    ]

    for tag in tags:
        add_tag(tag['title'], tag['description'], tag['colour'], tag['tag_id'], tag['team_id'])

    value_tags = [
        {
            'tag_id': '0',
            'team_id': '0000',
            'title': 'Test value tag',
            'description': "A value tag used for testing",
            'sub_values': [],
            'colour': '9fdddf'
        },
        {
            'tag_id': '1',
            'team_id': '0000',
            'title': 'Test value tag #2',
            'description': "A tag used for testing",
            'sub_values': [],
            'colour': 'f57f22'
        }
    ]

    for value_tag in value_tags:
        add_value_tag(value_tag['title'], value_tag['description'], value_tag['colour'], value_tag['tag_id'], value_tag['team_id'], value_tag['sub_values'])

    epics = [
        {
            'epic_id': '1',
            'epic_colour': 'c93434',
            'dashboard_id': '0',
            'title': 'Epic for testing',
            'order': 1,
            'values': [0,1],
            'last_edited_by': 'Niamh',
            'last_edited': datetime(2023, 10, 3, 15),
            'created_by': 'Niamh',
            'time_created': datetime(2023, 10, 3, 15),
        }
    ]

    for epic in epics:
        add_epic(epic['epic_id'], epic['epic_colour'], epic['dashboard_id'], epic['title'], epic['order'], epic['values'], epic['last_edited_by'],
                 epic['last_edited'], epic['created_by'], epic['time_created'])

    stories = [
        {
            'story_id': '1',
            'epic_id': '1',
            'title': 'Story for testing',
            'order': 1,
            'tags': [0],
            'user_story': 'As a developer, I want to have a test story, so I can use it in the react app',
            'definition_of_done': 'Test story should display on REST framework page',
            'values': [0],
            'story_points': 3,
            'priority': 'MEDIUM',
            'pairable': False,
            'assigned_to': 'Niamh',
            'state': 'In progress',
            'last_edited_by': 'Niamh',
            'last_edited': datetime(2023, 10, 3, 15),
            'created_by': 'Niamh',
            'time_created': datetime(2023, 10, 3, 15),
        }
    ]

    for story in stories:
        add_story(story['story_id'], story['epic_id'], story['title'], story['order'], story['tags'], story['user_story'],
                story['definition_of_done'], story['values'], story['story_points'], story['priority'],
                story['pairable'], story['assigned_to'], story['state'], story['last_edited_by'],
                story['last_edited'], story['created_by'], story['time_created'])

def add_sprint(sprint_id, dashboard_id, start_date, end_date, stories, story_list):
    sprint = Sprint.objects.get_or_create(id=sprint_id)[0]
    sprint.dashboard_id = dashboard_id
    sprint.start_date = start_date
    sprint.end_date = end_date
    sprint.stories.set(stories)
    sprint.story_list = story_list
    sprint.save()
    return sprint

def add_tracking_column(column_id, dashboard_id, team_id, title, mark_as_complete, stories, WIP):
    column = TrackingColumn.objects.get_or_create(id=column_id)[0]
    column.dashboard_id = dashboard_id
    column.team_id = team_id
    column.title = title
    column.mark_as_complete = mark_as_complete
    column.stories.set(stories)
    column.stories_list = str(stories)
    column.WIP = 10
    column.save()
    return column

def add_tag(title, description, colour, tag_id, team_id):
    tag = Tag.objects.get_or_create(id=tag_id)[0]
    tag.title = title
    tag.team_id = team_id
    tag.description = description
    tag.colour = colour
    tag.save()
    return tag

def add_value_tag(title, description, colour, tag_id, team_id, sub_values):
    tag = ValueTag.objects.get_or_create(id=tag_id)[0]
    tag.title = title
    tag.team_id = team_id
    tag.description = description
    tag.colour = colour
    tag.sub_values.set(sub_values)
    tag.save()
    return tag

def add_epic(epic_id, epic_colour, dashboard_id, title, order, values, last_edited_by, last_edited, created_by, time_created):
    epic = Epic.objects.get_or_create(epic_id = epic_id, dashboard_id=dashboard_id, epic_colour = epic_colour, order=order)[0]
    epic.title = title
    epic.last_edited_by = last_edited_by
    epic.last_edited = last_edited
    epic.created_by = created_by
    epic.time_created = time_created
    epic.values.set(values)
    epic.save()
    return epic

def add_story(story_id, epic_id, title, order, tags, user_story, definition_of_done, values, story_points, priority,
              pairable, assigned_to, state, last_edited_by, last_edited, created_by, time_created):
    story = Story.objects.get_or_create(story_id=story_id, epic_id=epic_id)[0]
    story.title = title
    story.order = order
    story.tags.set(tags)
    story.user_story = user_story
    story.definition_of_done = definition_of_done
    story.values.set(values)
    story.story_points = story_points
    story.priority = priority
    story.pairable = pairable
    story.assigned_to = assigned_to
    story.state = state
    story.last_edited_by = last_edited_by
    story.last_edited = last_edited
    story.created_by = created_by
    story.time_created = time_created
    story.save()
    return story

if __name__ == '__main__':
    print('Populating...')
    populate()


# Licensed under a 3-clause BSD style license - see LICENSE.rst

import functools
import numpy as np

from astropy.utils import iers
from astropy.tests.helper import pytest
from astropy.time import Time
from astropy.table import Table

try:
    import h5py  # pylint: disable=W0611  # noqa
except ImportError:
    HAS_H5PY = False
else:
    HAS_H5PY = True

try:
    import yaml  # pylint: disable=W0611  # noqa
    HAS_YAML = True
except ImportError:
    HAS_YAML = False

allclose_sec = functools.partial(np.allclose, rtol=2. ** -52,
                                 atol=2. ** -52 * 24 * 3600)  # 20 ps atol
is_masked = np.ma.is_masked

def test_simple():
    t = Time([1, 2, 3], format='cxcsec')
    assert t.masked is False
    assert np.all(t.mask == [False, False, False])

    # Before masking, format output is not a masked array (it is an ndarray
    # like always)
    assert not isinstance(t.value, np.ma.MaskedArray)
    assert not isinstance(t.unix, np.ma.MaskedArray)

    t[2] = np.ma.masked
    assert t.masked is True
    assert np.all(t.mask == [False, False, True])
    assert allclose_sec(t.value[:2], [1, 2])
    assert is_masked(t.value[2])
    assert is_masked(t[2].value)

    # After masking format output is a masked array
    assert isinstance(t.value, np.ma.MaskedArray)
    assert isinstance(t.unix, np.ma.MaskedArray)
    # Todo : test all formats

def test_scalar_init():
    t = Time('2000:001')
    assert t.masked is False
    assert t.mask == np.array(False)

def test_mask_not_writeable():
    t = Time('2000:001')
    with pytest.raises(AttributeError) as err:
        t.mask = True
    assert "can't set attribute" in str(err.value)

    t = Time(['2000:001'])
    with pytest.raises(ValueError) as err:
        t.mask[0] = True
    assert "assignment destination is read-only" in str(err.value)

def test_str():
    t = Time(['2000:001', '2000:002'])
    t[1] = np.ma.masked
    assert str(t) == "['2000:001:00:00:00.000' --]"
    assert repr(t) == "<Time object: scale='utc' format='yday' value=['2000:001:00:00:00.000' --]>"

    expected = ["masked_array(data=['2000-01-01 00:00:00.000', --],",
                '             mask=[False,  True],',
                "       fill_value='N/A',",
                "            dtype='<U23')"]

    # Note that we need to take care to allow for big-endian platforms,
    # for which the dtype will be >U23 instead of <U23, which we do with
    # the call to replace().
    assert repr(t.iso).replace('>U23', '<U23').splitlines() == expected

    # Assign value to unmask
    t[1] = '2000:111'
    assert str(t) == "['2000:001:00:00:00.000' '2000:111:00:00:00.000']"
    assert t.masked is False

def test_transform():
    with iers.conf.set_temp('auto_download', False):
        t = Time(['2000:001', '2000:002'])
        t[1] = np.ma.masked

        # Change scale (this tests the ERFA machinery with masking as well)
        t_ut1 = t.ut1
        assert is_masked(t_ut1.value[1])
        assert not is_masked(t_ut1.value[0])
        assert np.all(t_ut1.mask == [False, True])

        # Change format
        t_unix = t.unix
        assert is_masked(t_unix[1])
        assert not is_masked(t_unix[0])
        assert np.all(t_unix.mask == [False, True])

def test_masked_input():
    v0 = np.ma.MaskedArray([[1, 2], [3, 4]])  # No masked elements
    v1 = np.ma.MaskedArray([[1, 2], [3, 4]],
                           mask=[[True, False], [False, False]])
    v2 = np.ma.MaskedArray([[10, 20], [30, 40]],
                           mask=[[False, False], [False, True]])

    # Init from various combinations of masked arrays
    t = Time(v0, format='cxcsec')
    assert np.ma.allclose(t.value, v0)
    assert np.all(t.mask == [[False, False], [False, False]])
    assert t.masked is False

    t = Time(v1, format='cxcsec')
    assert np.ma.allclose(t.value, v1)
    assert np.all(t.mask == v1.mask)
    assert np.all(t.value.mask == v1.mask)
    assert t.masked is True

    t = Time(v1, v2, format='cxcsec')
    assert np.ma.allclose(t.value, v1 + v2)
    assert np.all(t.mask == (v1 + v2).mask)
    assert t.masked is True

    t = Time(v0, v1, format='cxcsec')
    assert np.ma.allclose(t.value, v0 + v1)
    assert np.all(t.mask == (v0 + v1).mask)
    assert t.masked is True

    t = Time(0, v2, format='cxcsec')
    assert np.ma.allclose(t.value, v2)
    assert np.all(t.mask == v2.mask)
    assert t.masked is True

    # Init from a string masked array
    t_iso = t.iso
    t2 = Time(t_iso)
    assert np.all(t2.value == t_iso)
    assert np.all(t2.mask == v2.mask)
    assert t2.masked is True

def test_all_masked_input():
    """Fix for #9612"""
    # Test with jd=0 and jd=np.nan. Both triggered an exception prior to #9624
    # due to astropy.utils.exceptions.ErfaError.
    for val in (0, np.nan):
        t = Time(np.ma.masked_array([val], mask=[True]), format='jd')
        assert str(t.iso) == '[--]'

def test_serialize_fits_masked(tmpdir):
    tm = Time([1, 2, 3], format='cxcsec')
    tm[1] = np.ma.masked

    fn = str(tmpdir.join('tempfile.fits'))
    t = Table([tm])
    t.write(fn)

    t2 = Table.read(fn, astropy_native=True)

    # Time FITS handling does not current round-trip format in FITS
    t2['col0'].format = tm.format

    assert t2['col0'].masked
    assert np.all(t2['col0'].mask == [False, True, False])
    assert np.all(t2['col0'].value == t['col0'].value)

@pytest.mark.skipif(not HAS_YAML or not HAS_H5PY,
                    reason='Need both h5py and yaml')
def test_serialize_hdf5_masked(tmpdir):
    tm = Time([1, 2, 3], format='cxcsec')
    tm[1] = np.ma.masked

    fn = str(tmpdir.join('tempfile.hdf5'))
    t = Table([tm])
    t.write(fn, path='root', serialize_meta=True)
    t2 = Table.read(fn)

    assert t2['col0'].masked
    assert np.all(t2['col0'].mask == [False, True, False])
    assert np.all(t2['col0'].value == t['col0'].value)

# Ignore warning in MIPS https://github.com/astropy/astropy/issues/9750
@pytest.mark.skipif('not HAS_YAML')
@pytest.mark.filterwarnings('ignore:invalid value encountered')
def test_serialize_ecsv_masked(tmpdir):
    tm = Time([1, 2, 3], format='cxcsec')
    tm[1] = np.ma.masked

    # Serializing in the default way for ECSV fails to round-trip
    # because it writes out a "nan" instead of "".  But for jd1/jd2
    # this works OK.
    tm.info.serialize_method['ecsv'] = 'jd1_jd2'

    fn = str(tmpdir.join('tempfile.ecsv'))
    t = Table([tm])
    t.write(fn)
    t2 = Table.read(fn)

    assert t2['col0'].masked
    assert np.all(t2['col0'].mask == [False, True, False])
    # Serializing floats to ASCII loses some precision so use allclose
    # and 1e-7 seconds tolerance.
    assert np.allclose(t2['col0'].value, t['col0'].value, rtol=0, atol=1e-7)


import sys
import logging
from unittest import mock
import galsim
import galsim.main  # Not imported automatically
import galsim.__main__
from galsim_test_helpers import *

# This file tests the galsim executable.
# Note: Most of the real functional tests are elsewhere.  E.g. test_config*
#       This really just tests the bits related specifically to the functions in galsim/main.py.

@timer
def test_args():
    """Test the argument parsing of galsim executable.
    """
    # no args or -h prints description and exits
    print('This should print the description...')  # Can only check by inspection, not assert
    with assert_raises(SystemExit):
        args = galsim.main.parse_args([])
    print('This should print the description...')
    with assert_raises(SystemExit):
        args = galsim.main.parse_args(['-h'])

    # --version prints version and exits
    print('This should print the version...')
    with assert_raises(SystemExit):
        args = galsim.main.parse_args(['--version'])

    # Any other options, but missing config file, also prints description and exits
    print('This should print the description...')
    with assert_raises(SystemExit):
        args = galsim.main.parse_args(['-v','3','-n','5'])

    # Normal operation requires a config_file parameter
    config_file = 'test.yaml'
    args = galsim.main.parse_args([config_file])
    assert args.config_file == config_file
    assert args.verbosity == 1
    assert args.log_file is None
    assert args.log_format == '%(message)s'
    assert args.file_type is None
    assert args.module is None
    assert args.profile is False
    assert args.njobs == 1
    assert args.job == 1
    assert args.except_abort is False
    assert args.version is False
    assert args.variables == []

    # Check setting each of those to be something else
    args = galsim.main.parse_args([config_file, '-v', '3'])
    assert args.config_file == config_file
    assert args.verbosity == 3

    args = galsim.main.parse_args([config_file, '-l', 'test.log'])
    assert args.config_file == config_file
    assert args.log_file == 'test.log'

    args = galsim.main.parse_args([config_file, '--log_format', '%(asctime)s - %(name)s - %(levelname)s - %(message)s'])
    assert args.config_file == config_file
    assert args.log_format == '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

    args = galsim.main.parse_args([config_file, '-f', 'yaml'])
    assert args.config_file == config_file
    assert args.file_type == 'yaml'
    args = galsim.main.parse_args([config_file, '-f', 'json'])
    assert args.config_file == config_file
    assert args.file_type == 'json'

    args = galsim.main.parse_args([config_file, '-m', 'des'])
    assert args.config_file == config_file
    assert args.module == ['des']
    args = galsim.main.parse_args([config_file, '-m', 'des', '-m', 'galsim_extra'])
    assert args.config_file == config_file
    assert args.module == ['des', 'galsim_extra']

    args = galsim.main.parse_args([config_file, '-p'])
    assert args.config_file == config_file
    assert args.profile is True

    args = galsim.main.parse_args([config_file, '-n', '3'])
    assert args.config_file == config_file
    assert args.njobs == 3

    args = galsim.main.parse_args([config_file, '-n', '3', '-j', '3'])
    assert args.config_file == config_file
    assert args.njobs == 3
    assert args.job == 3
    args = galsim.main.parse_args([config_file, '-n', '3', '-j', '1'])
    assert args.config_file == config_file
    assert args.njobs == 3
    assert args.job == 1

    args = galsim.main.parse_args([config_file, '-x'])
    assert args.config_file == config_file
    assert args.except_abort is True

    # --version with a config file doesn't exit, but still prints the version.
    print('This should print the version...')
    args = galsim.main.parse_args([config_file, '--version'])
    assert args.config_file == config_file

    # Additional arguments are accumulated in args.variables
    args = galsim.main.parse_args([config_file, 'output.nfiles=1', 'output.dir="."'])
    assert args.config_file == config_file
    assert args.variables == ['output.nfiles=1', 'output.dir="."']

    # Some invalid parameters
    with assert_raises(ValueError):
        galsim.main.parse_args([config_file, '-n', '0'])
    with assert_raises(ValueError):
        galsim.main.parse_args([config_file, '-n', '-1'])
    with assert_raises(ValueError):
        galsim.main.parse_args([config_file, '-n', '3', '-j', '4'])
    with assert_raises(ValueError):
        galsim.main.parse_args([config_file, '-n', '3', '-j', '0'])
    with assert_raises(ValueError):
        galsim.main.parse_args([config_file, '-j', '3'])

    # The ones handled by ArgumentParser raise SystemExit and print to stderr
    # To avoid ugly text output during nosetests runs, redirect stderr to stdout for a moment.
    sys_stderr = sys.stderr
    sys.stderr = sys.stdout
    with assert_raises(SystemExit):
        galsim.main.parse_args([config_file, '-f', 'invalid'])
    with assert_raises(SystemExit):
        galsim.main.parse_args([config_file, '-v', '-1'])
    with assert_raises(SystemExit):
        galsim.main.parse_args([config_file, '-v', '4'])
    sys.stderr = sys_stderr

# Need to call these before each time make_logger is repeated.  Else duplicate handles.
def remove_handler():
    logger = logging.getLogger('galsim')
    for handler in logger.handlers:
        logger.removeHandler(handler)

@timer
def test_logger():
    args = galsim.main.parse_args(['test.yaml'])

    remove_handler()
    logger = galsim.main.make_logger(args)
    assert logger.getEffectiveLevel() == logging.WARNING
    print('handlers = ',logger.handlers)
    print('This should print...')
    logger.warning("Test warning")
    print('This should not print...')
    logger.info("Test info")
    logger.debug("Test debug")
    print('Done')

    args.verbosity = 3
    args.log_file = 'output/test_logger.log'
    remove_handler()
    logger = galsim.main.make_logger(args)
    print('handlers = ',logger.handlers)
    assert logger.getEffectiveLevel() == logging.DEBUG
    logger.warning("Test warning")
    logger.info("Test info")
    logger.debug("Test debug")

    with open(args.log_file) as f:
        assert f.readline().strip() == "Test warning"
        assert f.readline().strip() == "Test info"
        assert f.readline().strip() == "Test debug"

    args.verbosity = 0
    remove_handler()
    logger = galsim.main.make_logger(args)
    print('handlers = ',logger.handlers)
    assert logger.getEffectiveLevel() == logging.ERROR
    logger.warning("Test warning")
    logger.info("Test info")
    logger.debug("Test debug")

    with open(args.log_file) as f:
        assert f.readlines() == []

    args.verbosity = 3
    args.log_format = '%(levelname)s - %(message)s'
    remove_handler()
    logger = galsim.main.make_logger(args)
    print('handlers = ',logger.handlers)
    assert logger.getEffectiveLevel() == logging.DEBUG
    logger.warning("Test warning")
    logger.info("Test info")
    logger.debug("Test debug")

    with open(args.log_file) as f:
        assert f.readline().strip() == "WARNING - Test warning"
        assert f.readline().strip() == "INFO - Test info"
        assert f.readline().strip() == "DEBUG - Test debug"

@timer
def test_parse_variables():
    logger = logging.getLogger('test_main')
    logger.setLevel(logging.ERROR)

    # Empty list -> empty dict
    new_params = galsim.main.parse_variables([], logger)
    assert new_params == {}

    vars = ["output.nfiles=1", "output.dir='.'"]
    new_params = galsim.main.parse_variables(vars, logger)
    assert new_params['output.nfiles'] == 1
    assert new_params['output.dir'] == '.'

    # Lists or dicts will be parsed here
    vars = ["psf={'type':'Gaussian','sigma':0.4}", "output.skip=[0,0,0,0,0,1]"]
    new_params = galsim.main.parse_variables(vars, logger)
    assert new_params['psf'] == {'type' : 'Gaussian', 'sigma' : 0.4}
    assert new_params['output.skip'] == [0,0,0,0,0,1]

    # Things that don't parse properly are just returned verbatim as a string
    # (Presumably they would give an appropriate error later when they are used if the
    # string is not a valid value for whatever this is.)
    vars = ["psf={'type':'Gaussian' : 'sigma':0.4}"]
    new_params = galsim.main.parse_variables(vars, logger)
    assert new_params['psf'] == "{'type':'Gaussian' : 'sigma':0.4}"

    # Missing = is an error
    vars = ["output.nfiles","1"]
    assert_raises(galsim.GalSimError, galsim.main.parse_variables, vars, logger)
    vars = ["output.nfiles-1"]
    assert_raises(galsim.GalSimError, galsim.main.parse_variables, vars, logger)

    # Should work correctly if yaml isn't available.
    # Although it doesn't always parse quite as nicely. E.g. requires ", not ' for string quotes.
    with mock.patch.dict(sys.modules, {'yaml':None}):
        vars = ['psf={"type":"Gaussian","sigma":0.4}', 'output.skip=[0,0,0,0,0,1]']
        new_params = galsim.main.parse_variables(vars, logger)
        assert new_params['psf'] == {'type' : 'Gaussian', 'sigma' : 0.4}
        assert new_params['output.skip'] == [0,0,0,0,0,1]

@timer
def test_modules():
    config = {}
    galsim.main.add_modules(config, None)
    assert config == {}

    galsim.main.add_modules(config, [])
    assert config == {}

    galsim.main.add_modules(config, ['des'])
    assert config == {'modules' : ['des']}

    galsim.main.add_modules(config, ['galsim_extra'])
    assert config == {'modules' : ['des', 'galsim_extra']}

    galsim.main.add_modules(config, ['a','b','c'])
    assert config == {'modules' : ['des', 'galsim_extra', 'a', 'b', 'c']}

@timer
def test_process():
    # Test running an extremely simple config, just testing a few features of how
    # the process_config() function works.
    config = {
        'gal' : {
            'type' : 'Gaussian',
            'sigma' : 2,
        },
        'output' : {
            'dir' : 'output',
            'file_name' : 'test_main.fits',
        }
    }
    file_name = os.path.join('output', 'test_main.fits')
    if os.path.exists(file_name):
        os.remove(file_name)
    args = galsim.main.parse_args(['test.yaml','-v','1'])
    remove_handler()
    logger = galsim.main.make_logger(args)

    galsim.main.process_config([config], args, logger)
    assert os.path.exists(file_name)
    assert config['root'] == 'test'  # This is set automatically
    args.profile = True
    print('Should print profile:')
    galsim.main.process_config([config], args, logger)
    assert config['profile'] is True
    print('Done')

    remove_handler()
    os.remove(file_name)
    config_file = os.path.join('input','test.yaml')
    galsim.main.main([config_file,'-v','1'])
    assert os.path.exists(file_name)

    with mock.patch('sys.argv', ['galsim', config_file, '-v', '1']):
        remove_handler()
        os.remove(file_name)
        galsim.main.run_main()
        assert os.path.exists(file_name)

        remove_handler()
        os.remove(file_name)
        galsim.__main__.run_main()
        assert os.path.exists(file_name)

if __name__ == "__main__":
    testfns = [v for k, v in vars().items() if k[:5] == 'test_' and callable(v)]
    for testfn in testfns:
        testfn()


#IMPORT CHROMEDRIVER(FOR REQUEST WEB), SELENIUM (SCRAPING) AND PANDAS (MANIPULATION DATA)
import selenium
from selenium import webdriver
import pandas as pd

#MAKING AN SERVICE TO GET WEB'S URL
#IMPORT KEYS
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.service import Service
import os
import time
import csv
import re

#GET URL
ser = Service("C:\\Users\chromedriver\chromedriver.exe")

#MAKE VARIABLE "OPTION"
option = webdriver.ChromeOptions()
option.add_argument("--incognito")
option.add_argument('--ignore-certificate-errors')

#OPEN BROWSER WITH WHITE PAGE (OPTION MODE)
browser = webdriver.Chrome(service=ser, options=option)

#DEFINE THE URL AND ACCESS IT
url = "https://www.autoscout24.com/lst/volkswagen/golf-(all)?sort=price&desc=0&ustate=N%2CU&kmto=50000&fregfrom=2016&atype=C&ac=0"
browser.get(url)
time.sleep(2)

#FIRST ITERATION AND MAKE URL LIST
i = 2
url_list = []

#APPEND FIRST URL TO URL_LIST
url_list.append(url)

#ITERATE AND APPEND TO URL_LIST
while i < 21:
    url_to_add = "https://www.autoscout24.com/lst/volkswagen/golf-(all)?sort=price&desc=0&ustate=N%2CU&size=20&page="+str(i)+"&kmto=50000&fregfrom=2016&atype=C&ac=0&"
    url_list.append(url_to_add)
    i += 1

#Print the resulting list(JUST CHECK THE FUNCTION OF URL_LIST)
#IF YOU WANT, YOU CAN DELETE THE HASHTAG

#print(url_list)

#IMPORT BEAUTIFULSOUP TO SCRAP THE WEB, BECAUSE IN MY PYTHON CAN'T USED SELENIUM ANYMORE
from bs4 import BeautifulSoup
import requests
results = []

#ITERATION TO SCRAPE EVERY CAR SPESIFICATION
no = 1
#ITERATION EVERY PAGE, THERE ARE ABOUT 20 CARS PER PAGE
for webpage in url_list:
    #REQUEST THE PAGE
    web = requests.get(webpage)

    #SOUP THE WEB
    soup = BeautifulSoup(web.content, "html.parser")
    results = soup.find_all("div", class_="cl-list-element cl-list-element-gap")
    #results = soup.body
    #print(len(results))
    #results.append(result)

    #ITERATION TO SCRAP DETAILS CAR SPESIFICATION
    for item_element in results:

        #MAKING LIST OF SPESIFICATION
        title_element = []
        price_element = []
        mileage_element = []
        spec_element = []

        #SCRAP THE (TEXT) WHETHER WE WANT
        title_element = item_element.find_all("h2","cldt-summary-version sc-ellipsis")
        price_element = item_element.find_all("div","cldt-summary-payment")
        #spec_element = item_element.find("ul")
        spec_element = item_element.find_all("li")

        #CLEANING THE RESULT OF SCRAPING
        for title in title_element:
            title = title.text.strip()
        for price in price_element:
            price = price.text.strip()

        #PRINT THE RESULT UNTIL - THE END -
        op = 1
        print(no,end='|')
        no += 1
        for spec in spec_element:
            if op == 1:
                print(title+'|'+price, end = '')
                op = 0
            print('|',end = spec.text.strip())
        print()


from azure.ai.textanalytics import TextAnalyticsClient
from azure.core.credentials import AzureKeyCredential

key = "ACCESS KEY"
endpoint = "ENDPOINT URL"

def authenticate_client():
    ta_credential = AzureKeyCredential(key)
    text_analytics_client = TextAnalyticsClient(
            endpoint=endpoint,
            credential=ta_credential)
    return text_analytics_client

client = authenticate_client()

def sentiment_analysis_example(client):

    documents = ["The movie was alright. The special effects were amazing. But the plot was not good."]
    response = client.analyze_sentiment(documents=documents)[0]
    print("Document Sentiment: {}".format(response.sentiment))
    print("Overall scores: positive={0:.2f}; neutral={1:.2f}; negative={2:.2f} \n".format(
        response.confidence_scores.positive,
        response.confidence_scores.neutral,
        response.confidence_scores.negative,
    ))
    for idx, sentence in enumerate(response.sentences):
        print("Sentence: {}".format(sentence.text))
        print("Sentence {} sentiment: {}".format(idx+1, sentence.sentiment))
        print("Sentence score:\nPositive={0:.2f}\nNeutral={1:.2f}\nNegative={2:.2f}\n".format(
            sentence.confidence_scores.positive,
            sentence.confidence_scores.neutral,
            sentence.confidence_scores.negative,
        ))

sentiment_analysis_example(client)


import sys

from PyQt5.QtWidgets import QApplication, QWidget, QDesktopWidget

def set_window_size():
    app = QApplication(sys.argv)

    w = QWidget()

    # 设置窗口标题
    w.setWindowTitle("演示窗口")

    # 窗口的大小
    w.resize(500, 300)

    # 将窗口设置在屏幕的左上角
    # w.move(0, 0)

    # 调整窗口在屏幕中央显示
    center_pointer = QDesktopWidget().availableGeometry().center()
    x = center_pointer.x()
    y = center_pointer.y()
    # w.move(x, y)
    # w.move(x-150, y-150)
    print(w.frameGeometry())
    print(w.frameGeometry().getRect())
    print(type(w.frameGeometry().getRect()))
    old_x, old_y, width, height = w.frameGeometry().getRect()
    w.move(x - width / 2, y - height / 2)

    # 展示窗口
    w.show()

    # 程序进行循环等待状态
    app.exec()

if __name__ == '__main__':
    set_window_size()

05 窗口显示在屏幕中央.py

05 窗口显示在屏幕中央.py


import requests
from bs4 import BeautifulSoup
import time
import re
import sqlite3

lista_paises = []

def banco():
    con = sqlite3.connect('dados.db')

    return con
def tabela():
    con =banco()

    cursor = con.cursor()

    try:

        cursor.execute('create table if not exists densidade('
                   'id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,'
                   'pais VARCHAR(50),'
                   'area DOUBLE,'
                   'populacao DOUBLE,'
                   'densidade VARCHAR(100));')

    except sqlite3.DataError as e:
        print(e)
    return con

def inserindo_dado(pais_vl,area_vl,populacao_vl,densidade_vl):
    con = tabela()
    cursor = con.cursor()

    cursor.execute("INSERT INTO densidade(pais,area,populacao,densidade) VALUES('"+pais_vl+"',+"+area_vl+",+"+populacao_vl+",+"+densidade_vl+")")
    con.commit()

    con.close()

def limpando_tabela():
    con = tabela()
    cursor = con.cursor()

    try:
        cursor.execute("DELETE FROM densidade")

    except sqlite3.DataError as e:
        print(e)
def page(url):
    page = None

    try:
        link = requests.get(url)

        page = link.text

    except requests.exceptions.RequestException as e:
        print('erro: ',e.reason)

    return page

def pegando_bs(link):
    pagina = page(link)
    bsObj = BeautifulSoup(pagina, 'html5lib')

    return bsObj

def pegando_links(bsObj):
    sites = []

    links = bsObj.find('tbody').findAll("a")
    for lista in links:
        if 'href' in lista.attrs:
            sites.append(lista.attrs['href'])

    return sites

def retorna_numero_area(valor):
    filtro = re.compile('([0-9]+)')
    num = valor.replace(",","")
    numero = filtro.findall((num))
    return numero[0]

def retorna_numero_populacao(valor):
    numero = valor.replace(",","")
    return numero

def retorna_densidade(populacao,area):
    populacao = int(populacao)
    area = float(area)
    if area==0:
        densidade=0
    else:
        densidade = populacao/area
    return densidade

def pegando_conteudo(subdominio):

    endereco = "http://example.webscraping.com/"+subdominio

    print(endereco)

    bsobj = pegando_bs(endereco)

    lis = pegando_links(bsobj)

    for link in lis:
        pagina = page("http://example.webscraping.com/" +link)
        bsObj = BeautifulSoup(pagina, 'html5lib')

        area = bsObj.find(attrs={"id":"places_area__row"}).find(attrs={"class":"w2p_fw"})
        populacao = bsObj.find(attrs={"id":"places_population__row"}).find(attrs={"class":"w2p_fw"})
        pais = bsObj.find(attrs={"id": "places_country__row"}).find(attrs={"class": "w2p_fw"})

        area_te = retorna_numero_area(area.text)
        populacao_te = retorna_numero_populacao(populacao.text)
        densidade =str(retorna_densidade(populacao_te,area_te))

        inserindo_dado(pais.text,area_te,populacao_te,densidade)

        dados={"pais":pais.text,
               "area":area_te,
               "populacao":populacao_te,
               "densidade_populacional":densidade,}
        lista_paises.append(dados)
        time.sleep(2)

    print(lista_paises)

    paginacao = bsobj.find(attrs={"id":"pagination"}).findAll('a')

    for pag in paginacao:

        if 'Next' in pag.text:
            ancora = pag.attrs['href']

            pegando_conteudo(ancora)

    return lista_paises
#limpando_tabela()
pegando_conteudo("places/default/index/0")


import os
import docx
import datetime
from datetime import datetime

def delete_row_in_table(document, search_text):
    for table in document.tables:
        for row in table.rows:
            for cell in row.cells:
                if search_text in cell.text:
                    table._tbl.remove(row._tr)
                    return True
    return False

def merge_docx_files(path):
    docx_files = [file for file in os.listdir(path) if file.endswith('.docx')]
    merged_doc = docx.Document()

    report_template = docx.Document(report_temp)
    report_date = datetime.now().strftime("%dth %B %Y")

    for table in report_template.tables:
        for row in table.rows:
            for cell in row.cells:
                #print("hello")
                if "DATE:" in cell.text:
                    #print(cell.text)
                    cell.text = cell.text.replace(cell.text, f"Date: {report_date}")
                #print(cell.text)
                merged_table = merged_doc.add_table(rows=0, cols=0)
                #merged_table._cells.append(table._cells)
                merged_table._element.append(table._element)

    for file in docx_files:
        doc = docx.Document(os.path.join(path, file))
        for table in doc.tables:
            for row in table.rows:
                if delete_row_in_table(doc, search_text):
                    merged_table = merged_doc.add_table(rows=0, cols=0)
                    merged_table._element.append(table._element)

    return merged_doc

directory_list = ["business_intelligence","business_performance","customer_service","internal_audit","process_development","reconciliation","revenue_assurance"]
op_directory = "combined_reports"
report_temp = "report_template.docx"
search_text = 'Date:'
current_directory = os.getcwd()

merged_document = merge_docx_files(current_directory)
current_date = datetime.now().strftime('%Y%m%d')
merged_filename = f'{current_date}.docx'
merged_document.save(os.path.join(current_directory, merged_filename))

Python Docx librery, read, write, append from docx.py


from datetime import datetime
from typing import Callable, Any, Union
import logging
import http.client

import sendgrid
from sendgrid.helpers.mail import Email, Content, Mail
import flask

from models.attributes import Attributes
from models.attribute_range import AttributeRange
from resources.alerts.push_alert import PushAlert
from models.alert_model import AlertWidgetModel
from models.users import Users
from models.sensor import Sensor
from models.location import Location
from app import create_app
from db import db
from settings.get_config_decorator import GetConfig

application = create_app()
logging.basicConfig(level='INFO')
logger = logging.getLogger(__name__)

def update_attribute_ranges(import_function: Callable) -> Callable:
    """
    Function decorator, track minimum and maximum value of all attributes
    :param import_function: Importer function
    :return: Importer function wrapped with minimum and maximum value
             functionality
    """
    def range_wrapper(*args: Any, **kwargs: dict):
        """
        Execute importer function and then compute minimum and maximum
        values of attributes
        :param args: Arguments of import_function parameter
        :param kwargs: Keyword Arguments of import_function parameter
        """
        import_function(*args, **kwargs)

        attribute_entries = Attributes.get_all()
        for attribute in attribute_entries:
            attribute_range = AttributeRange.get_by_attr_id(attribute.id)
            if attribute_range:
                most_recent_entry = Attributes.most_recent_timestamp(
                    attribute.table_name)
                if most_recent_entry:
                    if attribute_range.latest_update < most_recent_entry:
                        attr_min = Attributes.attribute_min(
                            attribute.table_name)
                        attr_max = Attributes.attribute_max(
                            attribute.table_name)
                        try:
                            attribute_range.minimum_sensor_id = \
                                attr_min.s_id
                            attribute_range.minimum = attr_min.value
                            attribute_range.minimum_recorded_date = \
                                attr_min.timestamp
                            attribute_range.maximum_sensor_id = \
                                attr_max.s_id
                            attribute_range.maximum = attr_max.value
                            attribute_range.maximum_recorded_date = \
                                attr_max.timestamp
                            attribute_range.latest_update = datetime.now()
                            attribute_range.save()
                            attribute_range.commit()

                            PushAlert.check_alerts(attribute_range)
                            check_min_and_max_alert_widgets(attribute_range)
                        except AttributeError:
                            pass
            else:
                attr_min = Attributes.attribute_min(attribute.table_name)
                attr_max = Attributes.attribute_max(attribute.table_name)
                try:
                    new_range_entry = \
                        AttributeRange(attribute.id,
                                       attr_min.s_id,
                                       attr_min.value,
                                       attr_min.timestamp,
                                       attr_max.s_id,
                                       attr_max.value,
                                       attr_max.timestamp,
                                       datetime.now())
                    new_range_entry.save()
                    new_range_entry.commit()
                    check_min_and_max_alert_widgets(new_range_entry)
                    PushAlert.check_alerts(new_range_entry)
                except AttributeError:
                    new_range_entry = AttributeRange(attribute.id, None, None,
                                                     None, None, None, None,
                                                     datetime.now())
                    new_range_entry.save()
                    new_range_entry.commit()

    return range_wrapper

def check_min_and_max_alert_widgets(attribute_range_entry: db.Model):
    """
    Send emails to users for alert widgets that have been triggered
    :param attribute_range_entry: Entry in the attribute range table
    """

    if attribute_range_entry.maximum:
        max_alerts = AlertWidgetModel.get_max_alerts(attribute_range_entry)
        for alert in max_alerts:
            user_details = Users.find_by_id(alert["user_id"])
            if user_details:
                attr = Attributes.get_by_id(attribute_range_entry.attribute_id)
                if attr:
                    if not send_alert_email(
                            user_details.email, user_details.fullname,
                            attr.name, attribute_range_entry.maximum,
                            attribute_range_entry.maximum_recorded_date,
                            attribute_range_entry.maximum_sensor_id,
                            alert["max_threshold"], "exceeded"):
                        logger.error("Server error prevented the  "
                                     "sending of a max alert email "
                                     "to {} regarding attribute with "
                                     "id {}".format(
                            user_details.email,
                            attribute_range_entry.attribute_id))
                else:
                    logger.error(
                        "Could not send max alert email to "
                        "user with id {} as the attribute with "
                        "id {} does not exist ".format(
                            alert["user_id"],
                            attribute_range_entry.attribute_id))
            else:
                logger.error("Could not send max alert email to "
                             "user with id {} as the user does "
                             "not exist ".format(alert["user_id"]))

    if attribute_range_entry.minimum:
        min_alerts = AlertWidgetModel.get_min_alerts(attribute_range_entry)
        for alert in min_alerts:
            user_details = Users.find_by_id(alert["user_id"])
            if user_details:
                attr = Attributes.get_by_id(attribute_range_entry.attribute_id)
                if attr:
                    if not send_alert_email(
                            user_details.email, user_details.fullname,
                            attr.name, attribute_range_entry.minimum,
                            attribute_range_entry.minimum_recorded_date,
                            attribute_range_entry.minimum_sensor_id,
                            alert["min_threshold"], "fell short of"):
                        logger.error("Server error prevented the sending of "
                                     "a  min alert email to {} regarding  "
                                     "attribute with id {}".format(
                            user_details.email,
                            attribute_range_entry.attribute_id))
                else:
                    logger.error(
                        "Could not send min alert email to "
                        "user with id {} as the attribute with "
                        "id {} does not exist ".format(
                            alert["user_id"],
                            attribute_range_entry.attribute_id))
            else:
                logger.error("Could not send min alert email to "
                             "user with id {} as the user does "
                             "not exist ".format(alert["user_id"]))

def send_alert_email(email: str, username: str, attribute: str,
                     value: Union[int,float], recorded_date: datetime,
                     sensor_id: str, threshold: Union[int, float],
                     alert_description: str) -> bool:

    """
    Send attribute alert email to a user
    :param email: User's email address
    :param username: User's name in the Users table
    :param attribute: Attribute name
    :param value: maximum value for the respective attribute
    :param recorded_date: Date which the maximum value was recorded
    :param sensor_id: ID of sensor which recorded the
    :param threshold: Value set by user to be checked
    :param alert_description: Describes whether an attribute has been
                              exceeded or fell short of the threshold value
    :return: Whether the alert email was sent
    """

    sensor = Sensor.get_by_id(sensor_id)
    sensor_lat, sensor_lon = None, None
    if sensor:
        loc = Location.get_by_id(sensor.l_id)
        if loc:
            sensor_lat = loc.lat
            sensor_lon = loc.lon
    else:
        logger.info("Could not include sensor information in alert "
                     "email as sensor with ID {} does not "
                     "exist".format(sensor_id))

    if alert_description == "exceeded":
        diff = value - threshold
    else:
        diff = threshold - value

    text_version = GetConfig.configure('alert', 'text_template').format(
        username=username, attribute=attribute, threshold=threshold,
        sensor_id=sensor_id, lat=sensor_lat, lon=sensor_lon,value=value,
        recorded_date=recorded_date, verb=alert_description, diff=diff)

    html_version = flask.render_template(
        GetConfig.configure('alert', 'html_template'), username=username,
        attribute=attribute, threshold=threshold, sensor_id=sensor_id,
        lat=sensor_lat, lon=sensor_lon,value=value,
        recorded_date=recorded_date, verb=alert_description, diff=diff)

    sg = sendgrid.SendGridAPIClient(apikey=GetConfig.configure('sendgrid',
                                                               'api_key'))

    from_email = Email(GetConfig.configure('alert', 'sender_email'))
    to_email = Email(email)
    subject = GetConfig.configure('alert', 'email_subject')
    content_text = Content("text/plain", text_version)
    send_alert = Mail(from_email, subject, to_email, content_text)
    content_html = Content("text/html", html_version)
    send_alert.add_content(content_html)

    try:
        email_response = sg.client.mail.send.post(
            request_body=send_alert.get())
        logger.info("Sent alert email to {} with "
                    "response code : {}".format(email,
                                                email_response.status_code))
        return True
    except http.client.IncompleteRead as e:
        logger.error("Sendgrid API Key may not be set correctly or be "
                     "invalid", e)
        return False


import os
from dataprovider.davis_cached_2016 import DataAccessHelper
import numpy as np
import matplotlib.pyplot as plt
from common.logger import getLogger
from skimage import io, transform
import skimage
from mpl_toolkits.axes_grid1 import make_axes_locatable
from PIL import Image

IMAGE_HEIGHT = 360
IMAGE_WIDTH = 480

davis = DataAccessHelper()
logger = getLogger()

def save_agg_vis_frame(out_seq_dir,seq_name,frame_no, src_dirs,src_frames,mean_frame,img):
    os.makedirs(out_seq_dir,exist_ok = True)
    out_fig_path = os.path.join(out_seq_dir,'{0:05}.png'.format(frame_no))
    src_dir_names = [name.split(os.sep)[-2] for name in src_dirs]
    fig = plt.figure(figsize=(16,10))

    ax = plt.subplot(2,3,1)
    plt.imshow(img)

    ax = plt.subplot(2,3,2)
    im = ax.imshow(mean_frame)
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im, cax=cax)

    ax = plt.subplot(2,3,4)
    im = ax.imshow(src_frames[0])
    ax.set_title(src_dir_names[0])
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im, cax=cax)

    ax = plt.subplot(2,3,5)
    im = ax.imshow(src_frames[1])
    ax.set_title(src_dir_names[1])
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im, cax=cax)

    ax = plt.subplot(2,3,6)
    im = ax.imshow(src_frames[2])
    ax.set_title(src_dir_names[2])
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im, cax=cax)

    fig.tight_layout()
    fig.savefig(out_fig_path, dpi=fig.dpi)
    plt.close(fig)

def threshold_image(prediction,threshold=0.5):
    prediction[prediction < 0.5] = 0
    prediction[prediction >= 0.5] = 1
    return prediction

def save_image(out_dir,sequence_name,frame_no,img):
    out_seq_dir = os.path.join(out_dir,sequence_name)
    os.makedirs(out_seq_dir,exist_ok = True)
    out_path = os.path.join(out_seq_dir,'{0:05}.png'.format(frame_no))
    io.imsave(out_path, img)

def mask_path(out_dir,seq_name, frame_no):
    label_path = os.path.join(out_dir,seq_name, '{0:05}.png'.format(frame_no))
    return label_path

def read_label(label_path):
    image = io.imread(label_path, as_grey=True)
    return image

def upper_bound_seq(src_dirs,sequence_name,out_dir):
    # Mask output dir
    mask_out_dir = os.path.join(out_dir, '480p')
    agg_vis_dir = os.path.join(out_dir, 'agg-vis',sequence_name)
    map_vis_dir = os.path.join(out_dir, 'map-vis',sequence_name)
    os.makedirs(map_vis_dir, exist_ok=True)

    palette_img = Image.open('/usr/stud/george/test.png')
    palette = palette_img.getpalette()

    frames = davis.all_frames_nums(sequence_name)
    label_path = davis.label_path(sequence_name, min(frames))
    prev_mask = davis.read_label(label_path) * 255
    save_image(mask_out_dir, sequence_name, min(frames), davis.read_label(label_path))

    for frame_no in range(min(frames) + 1, max(frames) + 1):
        print('computing seq:{} frame:{}'.format(sequence_name,frame_no))
        all_matched = []
        image_path = davis.image_path(sequence_name, frame_no)
        label_path = davis.label_path(sequence_name, frame_no)

        img = davis.read_image(image_path)
        gt_label = davis.read_label(label_path)

        gt_binary = (gt_label > 0)

        for sd in src_dirs:
            out_label_path = mask_path(sd,sequence_name,frame_no)
            out_label = read_label(out_label_path)
            out_binary = out_label > 0
            matched_binary  = np.logical_not(np.logical_xor(gt_binary,out_binary))
            all_matched.append(matched_binary)

        if len(src_dirs) == 2:
            matched_0 = all_matched[0]
            matched_1 = all_matched[1]
            matched_all = np.all(np.dstack(all_matched),axis=2)
            map = np.zeros(matched_0.shape,dtype=np.uint8)
            map[matched_0] = 1
            map[matched_1] = 2
            map[matched_all] = 3
            map_img = Image.fromarray(map,mode='P')
            map_img.putpalette(palette)
            map_path = os.path.join(map_vis_dir, '{0:05}.png'.format(frame_no))
            map_img.save(map_path)

        matched_any = np.any(np.dstack(all_matched),axis=2)
        mask_binary = np.select([matched_any,np.logical_not(matched_any)],[gt_binary,np.logical_not(gt_binary)])
        pred  = skimage.img_as_ubyte(mask_binary)*255
        #print(np.unique(pred))
        save_image(mask_out_dir, sequence_name, frame_no, pred)

def process_sequences(src_dirs,out_dir):
    test_sequences = davis.test_sequence_list()+davis.train_sequence_list()

    for seq in test_sequences:
        logger.info('aggregating results for sequence: {}'.format(seq))
        upper_bound_seq(src_dirs,seq,out_dir)

if __name__ == '__main__':

    out_dir = 'upperbound_osvos_and_tpsm-dist/'
    out_dir = os.path.join('../Results',out_dir)
    #src_dirs =['segnet480pvgg-wl-dp2-osvos-val-O0-4/iter-45000',
    #              'segnet480pvgg-wl-dp2-osvos-val-O1-3/iter-45000',
    #              ]

    #src_dirs = [os.path.join('test_out',sd,'480p') for sd in src_dirs]
    src_dirs = []
    #src_dirs.append("/usr/stud/george/workspace/adam/test_out/s480pvgg-davis2016-O1-osvosold-reg1e-4-mo<1e-2>-de-1/iter-500000/480p")
    #src_dirs.append("/usr/stud/george/workspace/adam/test_out/mergeosvosnet-seqdavis2016-B1O-1-adam<1e-6>-opt-adam-<1e-4>-3/iter-99000/480p")
    src_dirs.append("/usr/stud/george/workspace/adam/test_out/s480pvgg-segnet_brn-daviscombo-O1-Plabel_to_dist-osvosold-reg1e-4-mo<1e-2>-de-scale1.3-1/iter-685000/480p")

    src_dirs.append("/usr/stud/george/workspace/Results/OSVOS")

    process_sequences(src_dirs,out_dir)


# -*- coding: utf-8 -*-
"""
@Time: 2020/5/28 15:23
@Author: Sue Zhu
"""
from functools import partial

import numpy as np
import pandas as pd

from ..const import AssetEnum
from ..database import stock_
from ..interface import AbstractFactor
from ..utils.transformer import OutlierMAD, ScaleMinMax

class FamaFrench(AbstractFactor):
    """
    Classic Fama-French 3 Factor
    """
    asset_type = AssetEnum.STOCK
    start_date = pd.Timestamp('2003-12-31')

    def __init__(self):
        self.universe = stock_.StockUniverse()
        self.value_scale = ScaleMinMax(
            min_func=partial(np.nanpercentile, q=30, axis=0),
            max_func=partial(np.nanpercentile, q=70, axis=0)
        ).fit_transform

    @property
    def field_types(self):
        return dict(size=float, value=float, capt=float, label=str)

    def compute(self, dt):
        # stock match case
        universe = self.universe.get_instruments(dt)

        # get derivative data
        derivative = stock_.get_derivative_indicator(
            trade_dt=f'{dt:%Y%m%d}',
            fields=['mv', 'pe_ttm'],
        ).rename(columns={'mv': 'capt'}).filter(universe, axis=0).fillna({'pe_ttm': 0})

        # calculate and clean outliers.
        derivative['size'] = np.log10(derivative['capt'])
        derivative['value'] = 1 / derivative['pe_ttm']
        derivative.loc[:, ['size', 'value']] = OutlierMAD().fit_transform(derivative.loc[:, ['size', 'value']].values)

        # re scale data with robust-scale to make style identify easier
        derivative.loc[:, 'size'] = ScaleMinMax(
            min_func=lambda arr: derivative['size'].sort_values().iloc[-500],
            max_func=lambda arr: derivative['size'].sort_values().iloc[-200]
        ).fit_transform(derivative.loc[:, 'size']) * 200 - 100
        derivative['label'] = pd.cut(derivative['size'], bins=[-np.inf, -100, 100, np.inf], labels=list('SMB'))
        # value label should be neutralized by size
        derivative.loc[:, 'value'] = derivative.groupby('label')['value'].apply(self.value_scale) * 200 - 100
        derivative['label'] = derivative['label'].str.cat(
            pd.cut(derivative['value'], bins=[-np.inf, -100, 100, np.inf], labels=list('GNV'))
        )

        return derivative.filter(self.field_types.keys(), axis=1)


import zipfile as zf
import numpy as np
import os
import cv2
import scipy.misc

PATH = 'D:/landsat8/'

def extract_images_random(bands, mask, number, size = (250, 250)):
    minibands = np.empty((number, size[0], size[1], bands.shape[2]))
    masks = np.empty((number,size[0], size[1], 1))
    for k in range(number):
        while True:
            # check for correctness
            i = np.random.randint(0, bands.shape[0] - size[0])
            j = np.random.randint(0,bands.shape[1] - size[1])
            minibands[k, :, :, :] = bands[i: i + size[0], j : j + size[1], :]/255
            masks[k] = mask[i: i + size[0], j : j + size[1], :]/255
            if len(np.unique(masks[k])) > 1:
                break
    return minibands, masks

if __name__ == '__main__':
    nums = 10
    size = (256, 256)
    band_n = 7  # change band numb
    total = nums * 2
    minibands = np.empty((total, size[0], size[1], band_n))
    target = np.empty((total, size[0], size[1], 1))
    k = 0
    index = 0
    for value in os.listdir(PATH):
        if value == '8' or value == '7':
            path = PATH + value + '/' + value + '/'
            mask = cv2.imread(path + 'QB.tif')
            mask = mask[:, :, 1].reshape(mask.shape[0], mask.shape[1], 1)
            bnds = np.empty((mask.shape[0], mask.shape[1], band_n))
            for value in os.listdir(path):
                if value.split('.')[0][0] == 'B':
                    n = int(value.split('.')[0][1:])
                    if n < 8:
                        bnd = cv2.imread(path + value)[:, :, 0]
                        bnds[:, :, n - 1] = bnd
                        k += 1
            k = 0
            mnbands, msk = extract_images_random(bnds, mask, nums, size=size)
            minibands[index * nums: (index + 1) * nums, :, :, :] = mnbands
            target[index * nums: (index + 1) * nums, :, :, :] = msk
            index += 1

    save_path = PATH = 'D:/landsat8/' + 'valdata/'
    for i in range(total):
        scipy.misc.imsave(save_path + f'QB.tif', target[i].reshape((256, 256)))
        for k in range(band_n):
            scipy.misc.imsave(save_path + f'B{k + 1}.tif', minibands[i, :, :, k])

        with zf.ZipFile(save_path + f'val_data_{i}.zip', 'w') as zp:
            for file in os.listdir(save_path):
                if file.split('.')[1] == 'tif':
                    zp.write(save_path + file)


# sscbanlist

# Find all the bans made, then for each ban, try to find the post responsible
# Can take a date (YYYY-MM-DD) as a command line argument, in which case it will
# retrieve all bans after that date. If no argument is given, it will retrieve all
# bans it can.
#
# Originally authored by /u/marinuso

import datetime
import praw
import prawcore.exceptions as ex
import sys

### configuration ###

SUBREDDIT='slatestarcodex'

# scan comments this much time before a ban (in seconds)
COMMENT_TIME_WINDOW=3*24*60*60

# if two bans for the same user are issued shortly after each other,
# ignore the first ban (in seconds)
TIME_BETWEEN_BANS=12*60*60

# site name to use to read connection info from praw.ini
SITE_NAME='ssc'

# ignore bans whose description contains one of these
IGNORE=['stupid bot', 'bad bot', 'useless bot', 'spam']

# the format to output the bans in
#  Fields:
#   * {user} - the name of the banned user
#   * {date} - the date of the ban
#   * {duration} - the duration of the ban
#   * {description} - ban description (empty if not given)
#   * {comment} - Reddit link to user comment (empty if not found)
#   * {sample} - First non-quoted line of the user comment that has three
#                     or more words. This tends to help explain the ban. (empty if not found)
FORMAT='{date} | /u/{user} | {duration} | {comment} {sample}'

#####################

def main():
    timestamp = 0

    # take a date on the command line
    if len(sys.argv) == 2:
        timestamp = datetime.datetime.strptime(sys.argv[1],"%Y-%m-%d").timestamp()

    sb = SSCBanList()
    bans = sb.getBans(minTimestamp = timestamp)

    for ban in bans: print(ban.format(FORMAT))

class SSCBanList(object):
    """List all the bans of users in the subreddit, and for each ban,
    try to find the post that caused it (the post that had a mod response
    nearest to the ban)."""

    def __init__( self,
                   # defaults
                   subreddit=SUBREDDIT,
                   site_name=SITE_NAME,
                   # more arguments will be passed to praw.Reddit
                   **kwargs):
        """Connect to Reddit. subreddit may be changed, site_name may be
        emptied if you don't want to use praw.ini. Other arguments will
        be passed on to praw.Reddit."""

        # if someone manages to pass no subreddit, error
        if not subreddit: raise Exception("no subreddit")
        # only use the site name if it is set
        if site_name: kwargs['site_name'] = site_name

        self.reddit = praw.Reddit(user_agent="sscbanlist by /u/marinuso", **kwargs)
        self.reddit.user.me() # fail early if not logged in
        self.sub = self.reddit.subreddit(subreddit)

        self.user_comment_objs = {}
        self.user_comment_cache = {}

    def getBans(self, minTimestamp=0, window=COMMENT_TIME_WINDOW,
                  minDiff=TIME_BETWEEN_BANS, ignore=IGNORE):
        """Get all bans after a given timestamp."""
        banned_user_ts = {}
        bans = self.sub.mod.log(action='banuser', limit=None)
        r = []
        for ban in bans: # the bans are provided latest-first by the API
            # stop after minimum time reached
            if minTimestamp>ban.created_utc: break

            # disregard deleted users
            if ban.target_author == '[deleted]': continue

            # skip bans involving ignored reasons
            if any(x in (ban.description or []) for x in ignore): continue

            if ban.target_author in banned_user_ts \
            and banned_user_ts[ban.target_author]-ban.created_utc < minDiff:
                continue # disregard duplicate ban
            else:
                banned_user_ts[ban.target_author] = ban.created_utc

            sys.stderr.write("processing ban @%d: %s\n"%(ban.created_utc, ban.target_author))
            r.append(Ban(self, ban, window))
        return r

    def getUserComments(self, user):
        """Retrieve an user's comments (from cache if possible).
        Keep a list so it can be iterated over multiple times."""
        if not user in self.user_comment_cache:
            self.user_comment_cache[user] = []
            self.user_comment_objs[user] = user.comments.new(limit=None)

        idx = 0
        cache = self.user_comment_cache[user]
        obj = self.user_comment_objs[user]

        # next() will raise StopIteration and that's what breaks the loop
        while True:
            if idx >= len(cache):
                cache.append(next(obj))
            yield cache[idx]
            idx += 1

class Ban(object):
    """Represents a ban."""

    def __init__(self, sscbl, bandata, window):
        """Given a ModAction representing a ban, find the rest of the data."""
        self.sscbl = sscbl
        self.window = window

        # ban details
        self.duration = bandata.details
        self.description = bandata.description or ""
        self.timestamp = bandata.created_utc

        # banned user
        self.user = self.sscbl.reddit.redditor(bandata.target_author)

        # find the moderator comment
        try: (self.comment, self.mod_comment) = self.__findcomment()
        except (ex.NotFound, ex.Forbidden): # user may be deleted or suspended
            self.comment = None

    def format(self, fmt, datefmt="%Y-%m-%d", linktitle='Comment'):
        """Return formatted output for the ban.

        Fields:
         * {user} - the name of the banned user
         * {date} - the date of the ban
         * {duration} - the duration of the ban
         * {description} - ban description (empty if not given)
         * {comment} - Reddit link to moderator comment (empty if not found)
         * {sample} - First non-quoted line of the user comment that has three
                      or more words. This tends to help explain the ban. (empty if not found)

        The date will be formatted using the given format (using strftime).
        """

        if not self.comment: commentlink = sample = ''
        else:
            # comment link
            commentlink = '[{lt}](https://www.reddit.com/{link}?context=1)'.format(
                             lt=linktitle, link=self.comment.permalink)
            # try to extract a sample from the comment
            sample = ([
                line for line in self.comment.body.split("\n")
                     if line.strip()
                     and line.strip()[0]!='>'
                     and len(line.split())>=3] + [''])[0]

        return fmt.format(
                 user = self.user.name,
                 date = datetime.datetime.fromtimestamp(self.timestamp).strftime(datefmt),
                 duration = self.duration,
                 description = self.description,
                 comment = commentlink,
                 sample = sample)

    def __findcomment(self):
        """Find the offending comment."""

        # get the user's comments
        comments = self.sscbl.getUserComments(self.user)

        # the comments appear latest-first
        for comment in comments:
            # ignore comments created after ban
            if comment.created_utc > self.timestamp: continue
            # ignore comments outside the time window
            if comment.created_utc < self.timestamp-self.window: break
            # ignore comments not in the right subreddit
            if comment.subreddit != self.sscbl.sub: continue

            # check replies for distinguished moderator comment
            try:
                if len(comment.replies) == 0:
                    comment.refresh() # this sometimes fails, if that's the case don't give up
            except praw.exceptions.ClientException:
                continue

            replies = list(comment.replies)
            for reply in replies:
                if isinstance(reply, praw.models.MoreComments):
                    replies.append(reply.comments())
                    continue

                if reply.distinguished == 'moderator':
                    # found it
                    return (comment, reply)

        # didn't find it
        return (None, None)

if __name__=='__main__': main()


#!/usr/bin/python

from mininet.topo import Topo
from mininet.net import Mininet
from mininet.node import Node, OVSSwitch, Controller, RemoteController
from mininet.log import setLogLevel, info
from mininet.cli import CLI
import time
import os
import json
import string

class LinuxRouter( Node ):
    "A Node with IP forwarding enabled."

    def config( self, **params ):
        super( LinuxRouter, self).config( **params )
        # Enable forwarding on the router
        self.cmd( 'sysctl net.ipv4.ip_forward=1' )

    def terminate( self ):
        self.cmd( 'sysctl net.ipv4.ip_forward=0' )
        super( LinuxRouter, self ).terminate()

class NetworkTopo( Topo ):

    def build( self, **_opts ):

        data = getConfigFromJson(file_path("/addressConfiguration.json"))
        routers = {}
        routers = addRoutersToGraph(self,data)

        h1 = self.addHost( 'h1', ip='10.0.1.100/24', defaultRoute='via 10.0.1.10') #define gateway
        h2 = self.addHost( 'h2', ip='10.0.2.100/24', defaultRoute='via 10.0.2.20')

        s1 = self.addSwitch("s1", cls=OVSSwitch)

        addLinkBwRouters(self, data, routers)

        self.addLink(h1,routers["r1"],intfName2='r1-eth2',params2={ 'ip' : '10.0.1.10/24' })#params2 define the eth2 ip address
        self.addLink(h2,routers["r2"],intfName2='r2-eth2',params2={ 'ip' : '10.0.2.20/24' })

def addLinkBwRouters(self, data: dict, routers: dict):
    for firstInterface in data["links"]:
        secondInterface = data["links"][firstInterface]
        secondRouter = secondInterface.rpartition('-')[0]
        if (firstInterface.lower()[0] == "s"):
            firstRouter = firstInterface.rpartition('-')[0]
            secondInterface = data["links"][firstInterface]
            secondRouter = secondInterface.rpartition('-')[0]
            linkRouterWithRouter(self, firstRouter, secondRouter, firstInterface, secondInterface)
        else:
            firstRouter = firstInterface.rpartition('-')[0]
            info(" first, second device =  " + firstInterface+","+secondInterface + "\n")
            linkRouterWithSwitch(self, firstRouter, secondInterface, firstInterface)

def linkRouterWithRouter(self, firstRouter, secondRouter, firstInterface, secondInterface):
    self.addLink(firstRouter,secondRouter,intfName1=firstInterface,intfName2=secondInterface)

def linkRouterWithSwitch(self, router, switch, routerInterface):
    self.addLink(router, switch, intfName1=routerInterface)

def getConfigFromJson(path):
    with open(path, "r") as addressFile:
        data = json.load(addressFile)
    return data

def addRoutersToGraph(self, data: dict):
    routers = {}
    for index,router in enumerate(data["routers"]):
        interface = data["routers"][router]["interfaces"]["real"]
        routers["r" + str(index+1)] = self.addNode( router, cls=LinuxRouter, ip=interface[getFirstKeyOfDict(interface)] )
    return routers

def getFirstKeyOfDict(dataDict:dict):
    return list(dataDict.keys())[0]

def addAliasToInterface(interface: str, addressWithMask: str):
    # interface such as r1-eth1:0
    # addressWithMask such as 10.0.3.11/24
    return f'ifconfig {interface} {addressWithMask} \n'

def file_path(relative_path):
    # get correct absolute path for given file
    dir = os.path.dirname(os.path.abspath(__file__))
    split_path = relative_path.split("/")
    new_path = os.path.join(dir, *split_path)
    return new_path

def getRouterNames(data:dict):
    routers = []
    for index,router in enumerate(data["routers"]):
        routers.append(router)
    return routers

def loadZebraForAllRouters(net:Mininet, data:dict):
    routers = getRouterNames(data)
    for r in routers:
        device=net.getNodeByName(r)
        device.cmd(f'zebra -f conf/{r}zebra.conf -d -z ~/{r}zebra.api -i ~/{r}zebra.interface')

def loadOspfForAllRouters(net:Mininet, data:dict):
    routers = getRouterNames(data)
    for r in routers:
        device=net.getNodeByName(r)
        device.cmd(f'ospfd -f conf/{r}ospfd.conf -d -z ~/{r}zebra.api -i ~/{r}ospfd.interface')

def getTemplateOf(templateName:str):
    with open(file_path(f"/config_templates/{templateName}"), "r") as file:
        template = string.Template(file.read())
    return template

def generateZebraConfFIles(data:dict):
    zebraTemplate = getTemplateOf("zebra_template.conf").safe_substitute()
    routers = getRouterNames(data)
    for router in routers:
        with open(file_path(f'/conf/{router}zebra.conf'), 'w+') as filehandle:
            filehandle.write(zebraTemplate)

def getRouterFirstInterface(data:dict, router:str, withWildCard:bool):
    interface = data["routers"][router]["interfaces"]["real"]
    return interface[getFirstKeyOfDict(interface)] if withWildCard else removeWildCard(interface[getFirstKeyOfDict(interface)])

def getAllInterfacesOfRouter(data:dict, router:str):
    interface = data["routers"][router]["interfaces"]["real"]
    interfaceList = []
    for i in list(interface):
        interfaceList.append(i)
    return interfaceList

def getAllAddressesOfRouter(data:dict, router:str, withWildCard:bool):
    interface = data["routers"][router]["interfaces"]["real"]
    addressList = []
    for i in list(interface.keys()):
        addressList.append(interface[i]) if withWildCard else addressList.append(removeWildCard(interface[i]))
    return addressList

def removeWildCard(ip:str):
    return ip[:-3]

def zeroLastDigit(ip:str):
    index = ip.rindex(".")
    return ip[:index+1] + "0"

def generateOspfConfFiles(data:dict):
    routers = getRouterNames(data)
    ospfTemplate = getTemplateOf("ospf_template.conf")
    for router in routers:
        networkCommand = ""
        for address in getAllAddressesOfRouter(data, router, True):
            networkCommand += f"network {zeroLastDigit(address)}/24 area 0\n  "
        confFile = ospfTemplate.safe_substitute(
            name = f'{router}_ospf',
            id = getRouterFirstInterface(data, router, False),
            network = networkCommand)
        with open(file_path(f'/conf/{router}ospfd.conf'), 'w+') as filehandle:
            filehandle.write(confFile)

#TODO on the VM, remove old conf and use new ones
#TODO clean folders and improve readmes

def run():
    "Test linux router"
    topo = NetworkTopo()
    # add controller
    c0 = RemoteController('remoteController', ip = '192.168.1.78', port = 6653)
    net = Mininet(topo=topo , build=False, waitConnected=True, controller=RemoteController)
    net.addController(c0)

    net.build()
    net.start()

    data = getConfigFromJson(file_path("/addressConfiguration.json"))

    info('*** Starting switches, note: switch names must start with "s"\n')
    net.get('s1').start([c0])

    info('*** Starting controllers\n')
    for controller in net.controllers:
        controller.start()

    generateZebraConfFIles(data)
    generateOspfConfFiles(data)
    loadZebraForAllRouters(net, data)
    loadOspfForAllRouters(net, data)

    CLI( net )
    net.stop()
    os.system("killall -9 ospfd zebra")
    os.system("rm -f *api*")
    os.system("rm -f *interface*")

if __name__ == '__main__':
    setLogLevel( 'info' )
    run()


import PySimpleGUI as sg

sg.theme('Purple')

# loome loendid vastuste kokku arvutamiseks
valik_A = 0 #alati hea vastus
valik_B = 0 #alati halb vastus

# loome akna
# näide sulle kuidas tekst teistel küsimustel välja võiks näha
paigutus0 = [[sg.Graph(
            canvas_size =(800, 600),
            graph_bottom_left=(0, 800),
            graph_top_right=(800, 0),
            key="lõuend")],
            [sg.Button('START')],
            [sg.Cancel('Välju')]]

aken = sg.Window("Joonistame!", paigutus0)
aken.Finalize()
minu_lõuend = aken.Element("lõuend")
start = minu_lõuend.DrawImage(filename="start.png", location=(0, 0))
#taustapilt = minu_lõuend.DrawImage(filename="taustsober.png", location=(0, 0))

# loome tegevused
while True:
    syndmus0, v22rtused0 = aken.Read()

    # sulge programm, kui vajutatakse Välju või suletakse aken X-st
    if syndmus0 == "Välju" or syndmus0 == sg.WIN_CLOSED:
        break
    if syndmus0 == "START":
        aken.close()

        paigutus01 = [[sg.Graph(
            canvas_size =(800, 600),
            graph_bottom_left=(0, 800),
            graph_top_right=(800, 0),
            key="lõuend")],
            [sg.Text('Sul on valikus kolm erinevat mängu.')],
            [sg.Button('Suhted'), sg.Text('Saa teada, kas Sinu suhe on toksiline!')],
            [sg.Button('Sõbrad'), sg.Text('Saa teada, kuidas sõbrad Sind mõjutavad!')],
            [sg.Button('Sina'), sg.Text('Saa teada, kas Sul esineb probleeme!')],
            [sg.Cancel('Välju')]]

        aken = sg.Window("Joonistame!", paigutus01)
        aken.Finalize()
        minu_lõuend = aken.Element("lõuend")
        taustapilt = minu_lõuend.DrawImage(filename="valik1.png", location=(0, 0))

    syndmus01, v22rtused01 = aken.Read()

    if syndmus01 == "Välju" or syndmus01 == sg.WIN_CLOSED:
        break
    if syndmus01 == "Sõbrad":
        aken.close()

        paigutus1 = [[sg.Graph(
            canvas_size =(800, 600),
            graph_bottom_left=(0, 800),
            graph_top_right=(800, 0),
            key="lõuend")],
            [sg.Text('Sa ostad endale uued püksid ja näitad neid oma sõbrale.')],
            [sg.Text('Kuidas reageerib sellele su sõber?')],
            [sg.Button('A'), sg.Text('A: Oi kui ilusad püksid! Need sobivad sulle nii hästi!')],
            [sg.Button('B'), sg.Text('B: Iu kui kole sa välja näed. Sa peaksid need tagastama!')],
            [sg.Cancel('Välju')]]

        aken = sg.Window("Joonistame!", paigutus1)
        aken.Finalize()
        minu_lõuend = aken.Element("lõuend")
        taustapilt = minu_lõuend.DrawImage(filename="taustsober.png", location=(0, 0))

    syndmus1, v22rtused1 = aken.Read()

    # sulge programm, kui vajutatakse Välju või suletakse aken X-st
    if syndmus1 == "Välju" or syndmus1 == sg.WIN_CLOSED:
        break
    if syndmus1 == "A" or syndmus1 == "B":
        aken.close()

        if syndmus1 == "A":
            valik_A += 1
        if syndmus1 == "B":
            valik_B += 1

        # loome akna uue järjendiga
        paigutus2 = [[sg.Graph(
            canvas_size =(800, 600),
            graph_bottom_left=(0, 800),
            graph_top_right=(800, 0),
            key="lõuend")],
            [sg.Text('Sa oled oma sõbraga kohvikus ja mõtled mida tellida')],
            [sg.Text('Sina: "Hmm, ma tahan vist friikartulid tellida"')],
            [sg.Text('Kuidas vastab su sõber?')],
            [sg.Button('A'), sg.Text('See on väga hea mõte! Ma võtan sama.')],
            [sg.Button('B'), sg.Text('Sa peaksid küll salatit rohkem sööma. Vaata, milline sa välja näed!')],
            [sg.Cancel('Välju')]]

        aken = sg.Window("Joonistame!", paigutus2)
        aken.Finalize()
        minu_lõuend = aken.Element("lõuend")
        taustapilt = minu_lõuend.DrawImage(filename="taustsober.png", location=(0, 0))

        syndmus2, v22rtused2 = aken.read()

        if syndmus2 == "Välju" or syndmus2 == sg.WIN_CLOSED:
            break
        if syndmus2 == "A" or syndmus2 == "B":
            aken.close()

            if syndmus2 == "A":
                valik_A += 1
            if syndmus2 == "B":
                valik_B += 1

            # loome akna uue järjendiga
            paigutus3 = [[sg.Graph(
                canvas_size =(800, 600),
                graph_bottom_left=(0, 800),
                graph_top_right=(800, 0),
                key="lõuend")],
                [sg.Text('Sa pole oma sõpra 2 nädalat näinud ja helistad talle, et kokku saada.')],
                [sg.Text('Mida arvab sellest mõttest su sõber?')],
                [sg.Button('A'), sg.Text('Pole tõesti kaua näinud. Muidugi tahan ma sinuga kokku saada!')],
                [sg.Button('B'), sg.Text('Mul pole kogu aeg sinuga aega tegeleda. Ma saan sinu asemel täna hoopis mimmuga kokku.')],
                [sg.Cancel('Välju')]]

            aken = sg.Window("Joonistame!", paigutus3)
            aken.Finalize()
            minu_lõuend = aken.Element("lõuend")
            taustapilt = minu_lõuend.DrawImage(filename="taustsober.png", location=(0, 0))

            syndmus3, v22rtused3 = aken.read()

            if syndmus3 == "Välju" or syndmus3 == sg.WIN_CLOSED:
                break
            if syndmus3 == "A" or syndmus3 == "B":
                aken.close()

                if syndmus3 == "A":
                    valik_A += 1
                if syndmus3 == "B":
                    valik_B += 1

                # loome akna uue järjendiga
                paigutus4 = [[sg.Graph(
                    canvas_size =(800, 600),
                    graph_bottom_left=(0, 800),
                    graph_top_right=(800, 0),
                    key="lõuend")],
                    [sg.Text('Sa kukutad kogemata oma sõbra juures kausi maha ja see läheb katki.')],
                    [sg.Text('Kuidas reageerib su sõber?')],
                    [sg.Button('A'), sg.Text('Oi pole hullu! Ma aitan sul killud ära koristada.')],
                    [sg.Button('B'), sg.Text('Mis sul viga on! Kas sa oled mingi titt, kes ei suuda asju normaalselt käes hoida vä!?')],
                    [sg.Cancel('Välju')]]

                aken = sg.Window("Joonistame!", paigutus4)
                aken.Finalize()
                minu_lõuend = aken.Element("lõuend")
                taustapilt = minu_lõuend.DrawImage(filename="taustsober.png", location=(0, 0))

                syndmus4, v22rtused4 = aken.read()

                if syndmus4 == "Välju" or syndmus4 == sg.WIN_CLOSED:
                    break
                if syndmus4 == "A" or syndmus4 == "B":
                    aken.close()

                    if syndmus4 == "A":
                        valik_A += 1
                    if syndmus4 == "B":
                        valik_B += 1

                    # loome akna uue järjendiga
                    paigutus5 = [[sg.Graph(
                        canvas_size =(800, 600),
                        graph_bottom_left=(0, 800),
                        graph_top_right=(800, 0),
                        key="lõuend")],
                        [sg.Text('Sa sõidutad oma sõbra iga päev kooli ja koolist koju. Ühel päeval on su auto aga katki ja palud hoopis')],
                        [sg.Text('oma sõbralt sõitu.')],
                        [sg.Text('Mida vastab su sõber?')],
                        [sg.Button('A'), sg.Text('Muidugi võin ma su peale võtta! Sa sõidutad ju mind iga päev.')],
                        [sg.Button('B'), sg.Text('Ma ei viitsi sinu juurest läbi sõita. Vaata ise kuidas kooli saad.')],
                        [sg.Cancel('Välju')]]

                    aken = sg.Window("Joonistame!", paigutus5)
                    aken.Finalize()
                    minu_lõuend = aken.Element("lõuend")
                    taustapilt = minu_lõuend.DrawImage(filename="taustsober.png", location=(0, 0))

                    syndmus5, v22rtused5 = aken.read()

                    if syndmus5 == "Välju" or syndmus5 == sg.WIN_CLOSED:
                        break
                    if syndmus5 == "A" or syndmus5 == "B":
                        aken.close()

                        if syndmus5 == "A":
                            valik_A += 1
                        if syndmus5 == "B":
                            valik_B += 1

                    if str(valik_A) < str(valik_B):
                        paigutus6 = [[sg.Graph(
                            canvas_size =(800, 600),
                            graph_bottom_left=(0, 800),
                            graph_top_right=(800, 0),
                            key="lõuend")],
                            [sg.Text('Sinu sõber on halb')],
                            [sg.Text('Vastuseid A oli -'), sg.Text(str(valik_A))],
                            [sg.Text('Vastuseid B oli -'), sg.Text(str(valik_B))],
                            [sg.Cancel('Välju')]]

                        aken = sg.Window("Joonistame!", paigutus6)
                        aken.Finalize()
                        minu_lõuend = aken.Element("lõuend")
                        taustapilt = minu_lõuend.DrawImage(filename="taustsober.png", location=(0, 0))

                        syndmus6, v22rtused6 = aken.read()

                        if syndmus6 == "Välju" or syndmus6 == sg.WIN_CLOSED:
                            break

                    if str(valik_A) > str(valik_B):
                        paigutus7 = [[sg.Graph(
                            canvas_size =(800, 600),
                            graph_bottom_left=(0, 800),
                            graph_top_right=(800, 0),
                            key="lõuend")],
                            [sg.Text('Sinu sõber on hea')],
                            [sg.Text('Vastuseid A oli -'), sg.Text(str(valik_A))],
                            [sg.Text('Vastuseid B oli -'), sg.Text(str(valik_B))],
                            [sg.Cancel('Välju')]]

                        aken = sg.Window("Joonistame!", paigutus7)
                        aken.Finalize()
                        minu_lõuend = aken.Element("lõuend")
                        taustapilt = minu_lõuend.DrawImage(filename="taustsober.png", location=(0, 0))

                        syndmus7, v22rtused7 = aken.read()

                        if syndmus7 == "Välju" or syndmus7 == sg.WIN_CLOSED:
                            break

                        # loome akna uue järjendiga
#                         paigutus6 = [[sg.Graph(
#                             canvas_size =(800, 600),
#                             graph_bottom_left=(0, 800),
#                             graph_top_right=(800, 0),
#                             key="lõuend")],
#                             [sg.Text('Tulemused:')],
#                             [sg.Text('Vastuseid A oli -'), sg.Text(str(valik_A))],
#                             [sg.Text('Vastuseid B oli -'), sg.Text(str(valik_B))],
#                             [sg.Cancel('Välju')]]
#
#                         aken = sg.Window("Joonistame!", paigutus6)
#                         aken.Finalize()
#                         minu_lõuend = aken.Element("lõuend")
#                         taustapilt = minu_lõuend.DrawImage(filename="taustsober.png", location=(0, 0))
#
#                         syndmus6, v22rtused6 = aken.read()
#
#                         if syndmus6 == "Välju" or syndmus6 == sg.WIN_CLOSED:
#                             break

aken.close()


import requests
import pandas as pd
from lxml import etree

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.39"
}

def query_licence(SCID:str):
    url = "http://spscxkcx.gsxt.gov.cn/spscxk/detail.xhtml"

    params = {
        "zsbh":SCID,
        "lb":"SC"
    }
    req = requests.get(url,params=params,headers=headers)
    try:
        df = pd.read_html(req.text)
    except ValueError as e:
        return e
    return {i[0]:i[1] for i in list(df[0].values)}

def food_standards(standards):
    url = "http://down.foodmate.net/standard/search.php"
    params = {
        "corpstandard":2,
        "fields":0,
        "kw":standards
    }
    req = requests.get(url,params=params,headers=headers)
    html = etree.HTML(req.text)
    result = html.xpath('//*[@class="bgt"]//a/@title')
    return result

# food_standards("SB/T 10347")
# query_licence('SC12744030701096')
# html = etree.HTML(req.text)
# result = html.xpath('/html/body/div[2]/div/table/tbody/tr[5]/td/text()')    # 生产地址
# print(result)
# with open("test.html",mode="w",encoding="utf8") as f:
#     f.write(req.text)


from collections import namedtuple
import argparse
import cv2
import re
import time
import sys
import os
import numpy as np
from datetime import timedelta

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-g', '--geometry', required=True, type=str, help='Geometry in pixels where the dialog should appear (WxH+X+Y or "pick")')
    parser.add_argument('-i', '--input', required=True, type=str, help='Input video file or a still frame')
    parser.add_argument('-ss', '--start', default=0, type=int, help='Start from this frame')
    parser.add_argument('-n', '--threads', default=None, type=int, help='How many CPU threads to use. (Defaults to all)')
    parser.add_argument('output', nargs='?', type=str, help='Path to output detected timestamps. Defaults to stdout.', default=None)
    args = parser.parse_args()

    # state management
    config = {
        'mode': None,
        'geometry': None,
        'input': args.input,
        'output': args.output,
        'start': args.start,
        'threads': None
    }

    # special mode for picking the geometry from video using GUI
    if args.geometry == 'pick':
        config['mode'] = 'pick'
    else:
        # check if the geometry is in a valid format
        match = re.findall(r'(\d+)x(\d+)\+(\d+)\+(\d+)', args.geometry)
        if match is None or len(match) != 1 or len(match[0]) != 4:
            raise SyntaxError('Geometry argument is in an invalid format')
        width, height, x, y = [int(x) for x in match[0]]

        # ensure that it somewhat makes sense (logically)
        if x < 0 or y < 0 or width < 0 or height < 0:
            raise SyntaxError('Geometry values are invalid')

        config['mode'] = 'process'
        config['geometry'] = namedtuple('Geometry', ['width', 'height', 'x', 'y'])(width, height, x, y)
        config['threads'] = config['threads'] if config['threads'] is not None else os.cpu_count()

    # load the given video
    capture: cv2.VideoCapture = cv2.VideoCapture(config['input'])
    capture.set(cv2.CAP_PROP_POS_FRAMES, config['start'])
    vWidth, vHeight = (int(capture.get(cv2.CAP_PROP_FRAME_WIDTH)), int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT)))
    vFrames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))

    # if the mode is 'pick', just show a very simplistic "player" that displays pixel position
    if config['mode'] == 'pick':
        cv2.namedWindow('picker', cv2.WINDOW_NORMAL)

        # to drag and create the geometry
        def mouse_handler(event, x, y, flags, data):
            if event == cv2.EVENT_LBUTTONDOWN:
                data['start'] = (x, y)
                data['end'] = None
            elif event == cv2.EVENT_LBUTTONUP:
                data['end'] = (x, y)
                if data['start'][0] <= data['end'][0] and data['start'][1] <= data['end'][1]:
                    topleft = data['start']
                    bottomright = data['end']
                else:
                    topleft = data['end']
                    bottomright = data['start']
                print(f'{bottomright[0]-topleft[0]}x{bottomright[1]-topleft[1]}+{topleft[0]}+{topleft[1]}')
                data['clean'] = True

        # shared data for the mouse callback as well
        rect = { 'start': (0, 0), 'end': None, 'clean': False }
        cv2.setMouseCallback('picker', mouse_handler, rect)

        # slider for video position
        def position_change(position):
            capture.set(cv2.CAP_PROP_POS_FRAMES, position)
            rect['clean'] = True

        # note: we don't update this since (by default) it is quite taxing to do so
        cv2.createTrackbar('videopos', 'picker', 0, vFrames, position_change)

        playback = True
        while(capture.isOpened):
            if rect['clean']: # rect was changed, clean slate please
                ret, frame = capture.retrieve()
            if playback:
                framepos = int(capture.get(cv2.CAP_PROP_POS_FRAMES)) # loop instead of crashing pls
                if framepos == vFrames: capture.set(cv2.CAP_PROP_POS_FRAMES, 0)
                else: ret, frame = capture.read()
            if rect['end'] is not None: cv2.rectangle(frame, rect['start'], rect['end'], (0, 255, 0), 2)
            cv2.imshow('picker', frame)

            framepos = int(capture.get(cv2.CAP_PROP_POS_FRAMES))

            key = cv2.waitKey(1) & 0xFF
            if key == ord('q'):
                break
            elif key == ord(' '):
                playback = not playback
                cv2.setTrackbarPos('videopos', 'picker', framepos)
            elif key == 81:
                cv2.setTrackbarPos('videopos', 'picker', framepos - 1)
            elif key == 83:
                cv2.setTrackbarPos('videopos', 'picker', framepos + 1)
    elif config['mode'] == 'process':
        # validate geometry again
        if config['geometry'].width + config['geometry'].x > vWidth or config['geometry'].height + config['geometry'].y > vHeight:
            raise ValueError('Geometry is out of bounds from video')

        # set the output
        file = open(config['output'], 'w') if config['output'] else None
        if file is not None: file.write('frame;value\n') # CSV header

        # kernel for fixing compression bias
        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2,2))

        # debug outputting
        startTime = time.time()
        lastDebug = startTime
        lastPeriod = 30.0 # every 30 seconds
        lastProcessed = 0
        totalProcesed = config['start']

        frame = None
        freeze = False
        step = False

        # set CPU thread count
        cv2.setNumThreads(config['threads'])

        # deconstruct the geometry again...
        width, height, x, y = config['geometry']
        geometry_pixelarea = width * height

        # process per-frame
        while(capture.isOpened):
            framepos = totalProcesed # int(capture.get(cv2.CAP_PROP_POS_FRAMES))
            if framepos == vFrames: break

            if frame is None or step or not freeze: ret, frame = capture.read()
            if step: step = False

            # the "Save" button lights up brightly upon clicking, so checking every frame that it happens
            # Game is running on 30 FPS so technically we shouldn't miss it even with a 720p30 stream
            # The threshold values are 200, which is *very* lax, so I expect some incorrect detections.
            cropped = cv2.cvtColor(frame[
                y:y+height,
                x:x+width
            ], cv2.COLOR_BGR2HSV)
            # ret, gray = cv2.threshold(cv2.cvtColor(cropped, cv2.COLOR_RGB2GRAY), 200, 255, cv2.THRESH_BINARY)
            gray = cv2.inRange(cropped, (20, 20, 200), (40, 40, 255))
            # so the measured values RGB = 255, 30, 30 -> HSV?
            # so in BGR that'd be SVH...?

            # cv2.imshow('frame', np.vstack([cropped, cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR)]))

            # To fill in small pixel-gaps due to compression
            value = cv2.sumElems(cv2.dilate(gray, kernel))[0] / geometry_pixelarea

            # If the value is triggered, just print out the value
            if value >= 250.0:
                line = f'{framepos};{value}'
                if file is not None:
                    file.write(f'{line}\n')
                    file.flush()
                else: print(line)

            # Debug so we don't get insane
            lastProcessed = lastProcessed + 1
            totalProcesed = totalProcesed + 1
            currTime = time.time()
            if currTime - lastDebug >= lastPeriod:
                rate = float(lastProcessed) / lastPeriod
                eta = (vFrames - totalProcesed) / rate
                relative = float(totalProcesed) / vFrames * 100.0
                print(f'[DBG] ({relative:.2f}%) Processed {totalProcesed}/{vFrames} frames ({rate:.2f} FPS) (ETA: {str(timedelta(seconds=int(eta)))} seconds)', file=sys.stderr)
                lastProcessed = 0
                lastDebug = currTime

            # key = cv2.waitKey(1) & 0xFF
            # if key == ord('q'):
            #     break
            # if key == ord('d'):
            #     step = True
            # if key == ord(' '):
            #     freeze = not freeze
        if file: file.close()
        print(f'[DBG] Finished, it took {(time.time() - startTime):.2f} seconds', file=sys.stderr)

    # cleanup
    capture.release()
    cv2.destroyAllWindows()


from django.contrib import messages

from archivebox.search import query_search_index

class SearchResultsAdminMixin:
    def get_search_results(self, request, queryset, search_term: str):
        """Enhances the search queryset with results from the search backend"""

        qs, use_distinct = super().get_search_results(request, queryset, search_term)

        search_term = search_term.strip()
        if not search_term:
            return qs, use_distinct
        try:
            qsearch = query_search_index(search_term)
            qs = qs | qsearch
        except Exception as err:
            print(f'[!] Error while using search backend: {err.__class__.__name__} {err}')
            messages.add_message(request, messages.WARNING, f'Error from the search backend, only showing results from default admin search fields - Error: {err}')

        return qs, use_distinct


import translators.base_translator

class WorkspaceTranslator(translators.base_translator.BaseTranslator):
    """Translates an AWS Workspace to an IT Glue Configuration with an
    IT Glue Configuration Interface
    """
    FIELDS = [
        'name',
        'configuration_status_id',
        'notes',
        'ip_address',
        'ip_notes'
    ]

    def _name(self):
        return self.data.get('WorkspaceId') or 'Unnamed Workspace'

    def _configuration_status_id(self):
        state_name = self.data.get('State')
        active_status_id = self.options.get('active_status_id')
        inactive_status_id = self.options.get('inactive_status_id')
        if not active_status_id or not inactive_status_id:
            raise self.TranslatorError('Both an active_status_id and an inactive_status_id must be provided')
        if state_name.lower() == 'available':
            return active_status_id
        else:
            return inactive_status_id

    def _notes(self):
        workspace_props = self.data.get('WorkspaceProperties')
        notes_dict = {
            'workspace_id': self.data.get('WorkspaceId'),
            'computer_name': self.data.get('Computername'),
            'compute_type': workspace_props.get('ComputeTypeName'),
            'bundle_id': self.data.get('BundleId'),
            'subnet_id': self.data.get('SubnetId'),
            'directory_id': self.data.get('DirectoryId'),
            'error_code': self.data.get('ErrorCode'),
            'error_message': self.data.get('ErrorMessage'),
            'root_volume_size_gib': workspace_props.get('RootVolumeSizeGib'),
            'running_mode': workspace_props.get('RunningMode'),
            'running_mode_auto_stop_timeout_in_min': workspace_props.get('RunningModeAutoStopTimeoutInMinutes'),
            'user_volume_size_gib': workspace_props.get('UserVolumeSizeGib')
        }
        return self._format_notes(notes_dict)

    def _ip_address(self):
        return self.data.get('IpAddress', '')

    def _ip_notes(self):
        return self._format_notes({'subnet_id': self.data.get('SubnetId')})


# -*- coding: utf-8 -*-
"""
Created on Tue Nov 30 14:45:10 2021

@author: CSS
"""
from dotenv import load_dotenv
load_dotenv()

# Import additional required packages
import datetime as dt
import os
import requests
from google.cloud import storage
import json

import pandas as pd
from sodapy import Socrata

def main():
    print('Downloading crime data...')

    client = Socrata("data.cityofchicago.org", "bt7YFTw0UIlnWu0u0vmTbfbkA")
    results = client.get("ijzp-q8t2", limit=20000)

    # pd.set_option('display.max_columns', None)
    results_df = pd.DataFrame.from_records(results)
    # results_df.head()

    results = pd.DataFrame(results, columns=[
        'id', 'case_number', 'date', 'primary_type',
        'description', 'district', 'year', 'latitude',
        'longitude'])
    results.to_csv(f'crime_{dt.date.today()}.csv', index=False)

    # Upload local file of data to Google Cloud Storage
    print('Uploading crime data to GCS...')
    bucket_name = os.environ['PIPELINE_DATA_BUCKET']  # <-- retrieve the bucket name from the environment
    blob_name = f'crime_{dt.date.today()}.csv'
    outfile_path1 = f'crime_{dt.date.today()}.csv'

    storage_robot = storage.Client()
    bucket = storage_robot.bucket(bucket_name)
    blob = bucket.blob(blob_name)
    blob.upload_from_filename(outfile_path1)

    print('Done.')

    """
    from dotenv import load_dotenv
    load_dotenv()

    # Import additional required packages
    import datetime as dt
    import os
    import requests
    from google.cloud import storage
    import json
    import pandas as pd

    # Retrieve data from URL
    print('Downloading crime data...')
    response = requests.get('https://data.cityofchicago.org/resource/ijzp-q8t2.json')

    # Save retrieved data to a local file
    print('Saving crime data to a file...')

    outfile_path = f'crime_{dt.date.today()}.json'
    with open(outfile_path, mode='wb') as outfile:
        outfile.write(response.content)

    outfile_path1 = f'crime_{dt.date.today()}.csv'
    with open(outfile_path, "r") as read_file:
        data = pd.DataFrame(json.load(read_file))
        print(type(data))
        data.to_csv (outfile_path1, index = None)

    # Upload local file of data to Google Cloud Storage
    print('Uploading crime data to GCS...')
    bucket_name = os.environ['PIPELINE_DATA_BUCKET']  # <-- retrieve the bucket name from the environment
    blob_name = f'crime_{dt.date.today()}.csv'

    storage_robot = storage.Client()
    bucket = storage_robot.bucket(bucket_name)
    blob = bucket.blob(blob_name)
    blob.upload_from_filename(outfile_path1)

    print('Done.')
    """