import dbus
import os
import re
import time
import unittest
import six
import sys
import glob

from packaging.version import Version

import udiskstestcase

class UDisksLVMTestBase(udiskstestcase.UdisksTestCase):

    @classmethod
    def setUpClass(cls):
        udiskstestcase.UdisksTestCase.setUpClass()
        if not cls.check_module_loaded('lvm2'):
            udiskstestcase.UdisksTestCase.tearDownClass()
            raise unittest.SkipTest('Udisks module for LVM tests not loaded, skipping.')

    @classmethod
    def _get_lvm_version(cls):
        _ret, out = cls.run_command('lvm version')
        m = re.search(r'LVM version:.* ([\d\.]+)', out)
        if not m or len(m.groups()) != 1:
            raise RuntimeError('Failed to determine LVM version from: %s' % out)
        return Version(m.groups()[0])

    def _create_vg(self, vgname, devices):
        manager = self.get_object('/Manager')
        vg_path = manager.VolumeGroupCreate(vgname, devices, self.no_options,
                                            dbus_interface=self.iface_prefix + '.Manager.LVM2')
        vg = self.bus.get_object(self.iface_prefix, vg_path)
        self.assertIsNotNone(vg)
        # this makes sure the object is fully setup (e.g. has the Properties iface)
        vgsize = self.get_property(vg, '.VolumeGroup', 'Size')
        vgsize.assertGreater(0)
        ret, _out = self.run_command('vgs %s' % vgname)
        self.assertEqual(ret, 0)
        return vg

    def _remove_vg(self, vg, tear_down=False, ignore_removed=False):
        try:
            vgname = self.get_property_raw(vg, '.VolumeGroup', 'Name')
            if tear_down:
                options = dbus.Dictionary(signature='sv')
                options['tear-down'] = dbus.Boolean(True)
            else:
                options = self.no_options
            vg.Delete(True, options, dbus_interface=self.iface_prefix + '.VolumeGroup')
            ret, _out = self.run_command('vgs %s' % vgname)
            self.assertNotEqual(ret, 0)
        except dbus.exceptions.DBusException as e:
            if not ignore_removed:
                raise e

class UdisksLVMTest(UDisksLVMTestBase):
    '''This is a basic LVM test suite'''

    def _rescan_lio_devices(self):
        ''' Bring back all vdevs that have been deleted by the test '''
        ret, out = self.run_command("for f in $(find /sys/devices -path '*tcm_loop*/scan'); do echo '- - -' >$f; done")
        if ret != 0:
            raise RuntimeError("Cannot rescan vdevs: %s", out)
        self.udev_settle()
        # device names might have changed, need to find our vdevs again
        tcmdevs = glob.glob('/sys/devices/*tcm_loop*/tcm_loop_adapter_*/*/*/*/block/sd*')
        udiskstestcase.test_devs = self.vdevs = ['/dev/%s' % os.path.basename(p) for p in tcmdevs]
        for d in self.vdevs:
            obj = self.get_object('/block_devices/' + os.path.basename(d))
            self.assertHasIface(obj, self.iface_prefix + '.Block')

    def test_01_manager_interface(self):
        '''Test for module D-Bus Manager interface presence'''

        manager = self.get_object('/Manager')
        intro_data = manager.Introspect(self.no_options, dbus_interface='org.freedesktop.DBus.Introspectable')
        self.assertIn('interface name="%s.Manager.LVM2"' % self.iface_prefix, intro_data)

    def test_10_linear(self):
        '''Test linear (plain) LV functionality'''

        vgname = 'udisks_test_vg'

        # Use all the virtual devices but the last one
        devs = dbus.Array()
        for d in self.vdevs[:-1]:
            dev_obj = self.get_object('/block_devices/' + os.path.basename(d))
            self.assertIsNotNone(dev_obj)
            devs.append(dev_obj)
        vg = self._create_vg(vgname, devs)
        self.addCleanup(self._remove_vg, vg)

        dbus_vgname = self.get_property(vg, '.VolumeGroup', 'Name')
        dbus_vgname.assertEqual(vgname)

        # Create linear LV on the VG
        _ret, sys_vgsize = self.run_command('vgs -o size --noheadings --units=b --nosuffix %s' % vgname)
        vgsize = self.get_property(vg, '.VolumeGroup', 'Size')
        vgsize.assertEqual(int(sys_vgsize))

        _ret, sys_vgfree = self.run_command('vgs -o vg_free --noheadings --units=b --nosuffix %s' % vgname)
        vg_freesize = self.get_property(vg, '.VolumeGroup', 'FreeSize')
        vg_freesize.assertEqual(int(sys_vgfree))

        vg_freesize.assertEqual(vgsize.value)
        lvname = 'udisks_test_lv'
        lv_path = vg.CreatePlainVolume(lvname, dbus.UInt64(vgsize.value), self.no_options,
                                       dbus_interface=self.iface_prefix + '.VolumeGroup')
        self.assertIsNotNone(lv_path)

        ret, _out = self.run_command('lvs %s' % os.path.join(vgname, lvname))
        self.assertEqual(ret, 0)

        lv = self.bus.get_object(self.iface_prefix, lv_path)
        lv_block_path = lv.Activate(self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')
        self.assertIsNotNone(lv_block_path)
        lvsize = self.get_property(lv, '.LogicalVolume', 'Size')
        lvsize.assertEqual(vgsize.value)

        # check some dbus properties
        dbus_vg = self.get_property(lv, '.LogicalVolume', 'VolumeGroup')
        dbus_vg.assertEqual(str(vg.object_path))

        dbus_name = self.get_property(lv, '.LogicalVolume', 'Name')
        dbus_name.assertEqual(lvname)

        dbus_active = self.get_property(lv, '.LogicalVolume', 'Active')
        dbus_active.assertTrue()

        dbus_type = self.get_property(lv, '.LogicalVolume', 'Type')
        dbus_type.assertEqual('block')  # type is only 'block' or 'pool'

        dbus_layout = self.get_property(lv, '.LogicalVolume', 'Layout')
        dbus_layout.assertEqual('linear')

        def assertSegs(pvs):
            # Check that there is exactly one segment per PV
            struct = self.get_property(lv, '.LogicalVolume', 'Structure').value
            self.assertEqual(struct["type"], "linear")
            self.assertNotIn("data", struct)
            self.assertNotIn("metadata", struct)
            segs = struct["segments"]
            self.assertEqual(len(segs), len(pvs))
            seg_pvs = list(map(lambda s: s[2], segs))
            for p in pvs:
                self.assertIn(p.object_path, seg_pvs)

        assertSegs(devs)

        _ret, sys_uuid = self.run_command('lvs -o uuid --no-heading %s' % os.path.join(vgname, lvname))
        dbus_uuid = self.get_property(lv, '.LogicalVolume', 'UUID')
        dbus_uuid.assertEqual(sys_uuid)

        # check that the 'BlockDevice' property is set after Activate
        lv_prop_block = self.get_property(lv, '.LogicalVolume', 'BlockDevice')
        lv_prop_block.assertEqual(lv_block_path)

        # Shrink the LV
        lv.Resize(dbus.UInt64(lvsize.value/2), self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')

        lv_block = self.bus.get_object(self.iface_prefix, lv_block_path)
        self.assertIsNotNone(lv_block)
        new_lvsize = self.get_property(lv, '.LogicalVolume', 'Size')
        new_lvsize.assertLess(lvsize.value)

        # Add one more device to the VG
        new_dev_obj = self.get_object('/block_devices/' + os.path.basename(self.vdevs[-1]))
        self.assertIsNotNone(new_dev_obj)
        vg.AddDevice(new_dev_obj, self.no_options, dbus_interface=self.iface_prefix + '.VolumeGroup')
        new_vgsize = self.get_property(vg, '.VolumeGroup', 'Size')
        new_vgsize.assertGreater(vgsize.value)

        # Attempt to resize the LV to the whole VG, but specify only
        # the original PVS.  This is expected to fail.
        msg = "Insufficient free space"
        with six.assertRaisesRegex(self, dbus.exceptions.DBusException, msg):
            lv.Resize(dbus.UInt64(new_vgsize.value),
                      dbus.Dictionary({'pvs': devs}, signature='sv'),
                      dbus_interface=self.iface_prefix + '.LogicalVolume')

        # Now resize the LV to the whole VG without contraints
        lv.Resize(dbus.UInt64(new_vgsize.value), self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')
        new_lvsize = self.get_property(lv, '.LogicalVolume', 'Size')
        new_lvsize.assertEqual(new_vgsize.value)

        assertSegs(devs + [ new_dev_obj ])

        # rename the LV
        lvname = 'udisks_test_lv2'
        new_lvpath = lv.Rename(lvname, self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')

        # get the new (renamed) lv object
        lv = self.bus.get_object(self.iface_prefix, new_lvpath)
        self.assertIsNotNone(lv)

        dbus_name = self.get_property(lv, '.LogicalVolume', 'Name')
        dbus_name.assertEqual(lvname)

        ret, _out = self.run_command('lvs %s' % os.path.join(vgname, lvname))
        self.assertEqual(ret, 0)

        # deactivate/activate check
        dbus_prop_active = self.get_property(lv, '.LogicalVolume', 'Active')
        dbus_prop_active.assertTrue()
        ret, _out = self.run_command('lvchange %s --activate n' % os.path.join(vgname, lvname))
        self.assertEqual(ret, 0)
        time.sleep(3)
        dbus_prop_active = self.get_property(lv, '.LogicalVolume', 'Active')
        dbus_prop_active.assertFalse()
        ret, _out = self.run_command('lvchange %s --activate y' % os.path.join(vgname, lvname))
        self.assertEqual(ret, 0)
        time.sleep(3)
        dbus_prop_active = self.get_property(lv, '.LogicalVolume', 'Active')
        dbus_prop_active.assertTrue()
        lv.Deactivate(self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')
        dbus_prop_active = self.get_property(lv, '.LogicalVolume', 'Active')
        dbus_prop_active.assertFalse()

        # lvremove
        lv.Delete(self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')

        ret, _out = self.run_command('lvs %s' % os.path.join(vgname, lvname))
        self.assertNotEqual(ret, 0)

        # make sure the lv is not on dbus
        udisks = self.get_object('')
        objects = udisks.GetManagedObjects(dbus_interface='org.freedesktop.DBus.ObjectManager')
        self.assertNotIn(new_lvpath, objects.keys())

    @udiskstestcase.tag_test(udiskstestcase.TestTags.UNSTABLE)
    def test_15_raid(self):
        '''Test raid volumes functionality'''

        vgname = 'udisks_test_vg'

        # Use all the virtual devices
        devs = dbus.Array()
        for d in self.vdevs:
            dev_obj = self.get_object('/block_devices/' + os.path.basename(d))
            self.assertIsNotNone(dev_obj)
            devs.append(dev_obj)
            self.addCleanup(self.wipe_fs, d)
        vg = self._create_vg(vgname, devs)
        self.addCleanup(self._remove_vg, vg)

        dbus_vgname = self.get_property(vg, '.VolumeGroup', 'Name')
        dbus_vgname.assertEqual(vgname)

        first_vdev_uuid = self.get_property(devs[0], '.Block', 'IdUUID').value

        # Create raid1 LV on the VG
        lvname = 'udisks_test_lv'
        vg_freesize = self.get_property(vg, '.VolumeGroup', 'FreeSize')
        vdev_size = vg_freesize.value / len(devs)
        lv_size = int(vdev_size * 0.75)
        lv_path = vg.CreatePlainVolumeWithLayout(lvname, dbus.UInt64(lv_size),
                                                 "raid1", devs[0:3],
                                                 self.no_options,
                                                 dbus_interface=self.iface_prefix + '.VolumeGroup')
        self.assertIsNotNone(lv_path)

        lv = self.bus.get_object(self.iface_prefix, lv_path)
        self.get_property(lv, '.LogicalVolume', 'SyncRatio').assertEqual(1.0, timeout=60, poll_vg=vg)

        _ret, sys_type = self.run_command('lvs -o seg_type --noheadings --nosuffix %s/%s' % (vgname, lvname))
        self.assertEqual(sys_type, "raid1")
        self.get_property(lv, '.LogicalVolume', 'Layout').assertEqual("raid1")

        def assertSegs(struct, size, pv):
            self.assertEqual(struct["type"], "linear")
            self.assertNotIn("data", struct)
            self.assertNotIn("metadata", struct)
            if pv is not None:
                self.assertEqual(len(struct["segments"]), 1)
                if size is not None:
                    self.assertEqual(struct["segments"][0][1], size)
                self.assertEqual(struct["segments"][0][2], pv.object_path)
            else:
                self.assertEqual(len(struct["segments"]), 0)

        def assertRaid1Stripes(structs, size, pv1, pv2, pv3):
            self.assertEqual(len(structs), 3)
            assertSegs(structs[0], size, pv1)
            assertSegs(structs[1], size, pv2)
            assertSegs(structs[2], size, pv3)

        def assertRaid1Structure(pv1, pv2, pv3):
            struct = self.get_property(lv, '.LogicalVolume', 'Structure').value
            self.assertEqual(struct["type"], "raid1")
            self.assertEqual(struct["size"], lv_size)
            self.assertNotIn("segments", struct)
            assertRaid1Stripes(struct["data"], lv_size, pv1, pv2, pv3)
            assertRaid1Stripes(struct["metadata"], None, pv1, pv2, pv3)

        def waitRaid1Structure(pv1, pv2, pv3):
            for _ in range(5):
                try:
                    assertRaid1Structure(pv1, pv2, pv3)
                    return
                except AssertionError:
                    pass
                time.sleep(1)
            # Once again for the error message
            assertRaid1Structure(pv1, pv2, pv3)

        waitRaid1Structure(devs[0], devs[1], devs[2])

        # Yank out the first vdev and repair the LV with the fourth
        _ret, _output = self.run_command('echo yes >/sys/block/%s/device/delete' % os.path.basename(self.vdevs[0]))
        self.addCleanup(self._rescan_lio_devices)
        # give udisks some time to register the change
        self.run_command('udevadm trigger %s' % self.vdevs[0])
        self.udev_settle()
        _ret, sys_health = self.run_command('lvs -o health_status --noheadings --nosuffix %s/%s' % (vgname, lvname))
        self.assertEqual(sys_health, "partial")

        waitRaid1Structure(None, devs[1], devs[2])
        self.get_property(vg, '.VolumeGroup', 'MissingPhysicalVolumes').assertEqual([first_vdev_uuid])

        lv.Repair(devs[3:4], self.no_options,
                  dbus_interface=self.iface_prefix + '.LogicalVolume')
        _ret, sys_health = self.run_command('lvs -o health_status --noheadings --nosuffix %s/%s' % (vgname, lvname))
        self.assertEqual(sys_health, "")
        self.get_property(lv, '.LogicalVolume', 'SyncRatio').assertEqual(1.0, timeout=60, poll_vg=vg)

        waitRaid1Structure(devs[3], devs[1], devs[2])

        # Tell the VG that everything is alright
        vg.RemoveMissingPhysicalVolumes(self.no_options,
                                        dbus_interface=self.iface_prefix + '.VolumeGroup')
        self.get_property(vg, '.VolumeGroup', 'MissingPhysicalVolumes').assertEqual([])

    def test_20_thin(self):
        '''Test thin volumes functionality'''

        vgname = 'udisks_test_thin_vg'

        # Use all the virtual devices
        devs = dbus.Array()
        for d in self.vdevs:
            dev_obj = self.get_object('/block_devices/' + os.path.basename(d))
            self.assertIsNotNone(dev_obj)
            devs.append(dev_obj)
        vg = self._create_vg(vgname, devs)
        self.addCleanup(self._remove_vg, vg)

        # Create thin pool on the VG
        vgsize = int(self.get_property_raw(vg, '.VolumeGroup', 'FreeSize'))
        tpname = 'udisks_test_tp'
        tp_path = vg.CreateThinPoolVolume(tpname, dbus.UInt64(vgsize), self.no_options,
                                          dbus_interface=self.iface_prefix + '.VolumeGroup')
        self.assertIsNotNone(tp_path)

        ret, _out = self.run_command('lvs %s' % os.path.join(vgname, tpname))
        self.assertEqual(ret, 0)

        tp = self.bus.get_object(self.iface_prefix, tp_path)
        tpsize = self.get_property(tp, '.LogicalVolume', 'Size')

        # check that we report same size as lvs (udisks includes metadata so we need to add it too)
        _ret, dsize = self.run_command('lvs -olv_size --noheadings --units=b --nosuffix %s' % os.path.join(vgname, tpname))
        _ret, msize = self.run_command('lvs -olv_metadata_size --noheadings --units=b --nosuffix %s' % os.path.join(vgname, tpname))
        tpsize.assertEqual(int(dsize.strip()) + int(msize.strip()))

        dbus_type = self.get_property(tp, '.LogicalVolume', 'Type')
        dbus_type.assertEqual("pool")

        # Create thin volume in the pool with virtual size twice the backing pool
        tvname = 'udisks_test_tv'
        tv_path = vg.CreateThinVolume(tvname, dbus.UInt64(int(tpsize.value) * 2), tp, self.no_options,
                                      dbus_interface=self.iface_prefix + '.VolumeGroup')
        tv = self.bus.get_object(self.iface_prefix, tv_path)
        self.assertIsNotNone(tv)

        ret, _out = self.run_command('lvs %s' % os.path.join(vgname, tvname))
        self.assertEqual(ret, 0)

        # Check the block device of the thin volume
        lv_block_path = tv.Activate(self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')
        lv_block = self.bus.get_object(self.iface_prefix, lv_block_path)
        self.assertIsNotNone(lv_block)
        blocksize = self.get_property(lv_block, '.Block', 'Size')
        blocksize.assertGreater(vgsize)

        tv_tp = self.get_property(tv, '.LogicalVolume', 'ThinPool')
        tv_tp.assertEqual(tp_path)

    def test_30_snapshot(self):
        '''Test LVM snapshoting'''

        vgname = 'udisks_test_snap_vg'

        # Use all the virtual devices
        devs = dbus.Array()
        for d in self.vdevs:
            dev_obj = self.get_object('/block_devices/' + os.path.basename(d))
            self.assertIsNotNone(dev_obj)
            devs.append(dev_obj)
        vg = self._create_vg(vgname, devs)
        self.addCleanup(self._remove_vg, vg)

        # Create the origin LV
        vgsize = int(self.get_property_raw(vg, '.VolumeGroup', 'FreeSize'))
        lvname = 'udisks_test_origin_lv'
        lv_path = vg.CreatePlainVolume(lvname, dbus.UInt64(vgsize / 2), self.no_options,
                                       dbus_interface=self.iface_prefix + '.VolumeGroup')
        lv = self.bus.get_object(self.iface_prefix, lv_path)
        self.assertIsNotNone(lv)

        ret, _out = self.run_command('lvs %s' % os.path.join(vgname, lvname))
        self.assertEqual(ret, 0)

        # Create the LV's snapshot
        snapname = 'udisks_test_snap_lv'
        vg_freesize = int(self.get_property_raw(vg, '.VolumeGroup', 'FreeSize'))
        snap_path = lv.CreateSnapshot(snapname, vg_freesize, self.no_options,
                                      dbus_interface=self.iface_prefix + '.LogicalVolume')
        snap = self.bus.get_object(self.iface_prefix, snap_path)
        self.assertIsNotNone(snap)

        # check dbus properties
        dbus_origin = self.get_property(snap, '.LogicalVolume', 'Origin')
        dbus_origin.assertEqual(lv_path)

        dbus_name = self.get_property(snap, '.LogicalVolume', 'Name')
        dbus_name.assertEqual(snapname)

        ret, _out = self.run_command('lvs %s' % os.path.join(vgname, snapname))
        self.assertEqual(ret, 0)

    def test_40_cache(self):
        '''Basic LVM cache test'''

        vgname = 'udisks_test_cache_vg'

        # Use all the virtual devices
        devs = dbus.Array()
        for d in self.vdevs:
            dev_obj = self.get_object('/block_devices/' + os.path.basename(d))
            self.assertIsNotNone(dev_obj)
            devs.append(dev_obj)
        vg = self._create_vg(vgname, devs)
        self.addCleanup(self._remove_vg, vg)

        # Create the origin LV
        vgsize = int(self.get_property_raw(vg, '.VolumeGroup', 'FreeSize'))
        orig_lvname = 'udisks_test_origin_lv'
        lv_path = vg.CreatePlainVolume(orig_lvname, dbus.UInt64(vgsize / 2), self.no_options,
                                       dbus_interface=self.iface_prefix + '.VolumeGroup')
        lv = self.bus.get_object(self.iface_prefix, lv_path)
        self.assertIsNotNone(lv)

        ret, _out = self.run_command('lvs %s' % os.path.join(vgname, orig_lvname))
        self.assertEqual(ret, 0)

        # Create the caching LV
        cache_lvname = 'udisks_test_cache_lv'
        vgsize = int(self.get_property_raw(vg, '.VolumeGroup', 'FreeSize'))
        # 8 MiB reserved for the cache metadata created automatically by LVM
        lv_cache_path = vg.CreatePlainVolume(cache_lvname, dbus.UInt64((vgsize / 2) - 8 * 1024**2), self.no_options,
                                             dbus_interface=self.iface_prefix + '.VolumeGroup')
        cache_lv = self.bus.get_object(self.iface_prefix, lv_cache_path)
        self.assertIsNotNone(cache_lv)

        # Add the cache to the origin
        lv.CacheAttach('udisks_test_cache_lv', self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')

        _ret, out = self.run_command('lvs %s/%s --noheadings -o segtype' % (vgname, orig_lvname))
        self.assertEqual(out, 'cache')

        # Split the cache
        lv.CacheSplit(self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')

        _ret, out = self.run_command('lvs %s/%s --noheadings -o lv_layout' % (vgname, orig_lvname))
        self.assertEqual(out, 'linear')

        _ret, out = self.run_command('lvs %s/%s --noheadings -o lv_layout' % (vgname, cache_lvname))
        self.assertEqual(out, 'cache,pool')

    def test_50_rename_vg(self):
        ''' Test VG renaming '''

        vgname = 'udisks_test_rename_vg'

        # Use all the virtual devices
        devs = dbus.Array()
        for d in self.vdevs:
            dev_obj = self.get_object('/block_devices/' + os.path.basename(d))
            self.assertIsNotNone(dev_obj)
            devs.append(dev_obj)

        vg = self._create_vg(vgname, devs)

        vgname = 'udisks_test_rename_vg2'
        new_vgpath = vg.Rename(vgname, self.no_options, dbus_interface=self.iface_prefix + '.VolumeGroup')

        # get the new (renamed) lv object
        vg = self.bus.get_object(self.iface_prefix, new_vgpath)
        self.assertIsNotNone(vg)
        self.addCleanup(self._remove_vg, vg)

        dbus_name = self.get_property(vg, '.VolumeGroup', 'Name')
        dbus_name.assertEqual(vgname)

        ret, _out = self.run_command('vgs %s' % vgname)
        self.assertEqual(ret, 0)

    def test_60_pvs(self):
        ''' Test adding and removing PVs from VG '''

        vgname = 'udisks_test_pv_vg'

        # create vg with one pv
        old_pv = self.get_object('/block_devices/' + os.path.basename(self.vdevs[0]))
        self.assertIsNotNone(old_pv)

        vg = self._create_vg(vgname, dbus.Array([old_pv]))
        self.addCleanup(self._remove_vg, vg)

        # create an lv on it
        lvname = 'udisks_test_lv'
        lv_path = vg.CreatePlainVolume(lvname, dbus.UInt64(4 * 1024**2), self.no_options,
                                       dbus_interface=self.iface_prefix + '.VolumeGroup')
        lv = self.bus.get_object(self.iface_prefix, lv_path)
        self.assertIsNotNone(lv)

        # add a new pv to the vg
        new_pv = self.get_object('/block_devices/' + os.path.basename(self.vdevs[1]))
        vg.AddDevice(new_pv, self.no_options, dbus_interface=self.iface_prefix + '.VolumeGroup')

        _ret, out = self.run_command('pvs --noheadings -o vg_name %s' % self.vdevs[1])
        self.assertEqual(out, vgname)

        # empty the old pv
        vg.EmptyDevice(old_pv, self.no_options, dbus_interface=self.iface_prefix + '.VolumeGroup', timeout=120 * 100)

        _ret, pv_size = self.run_command('pvs --noheadings --units=B --nosuffix -o pv_size %s' % self.vdevs[0])
        _ret, pv_free = self.run_command('pvs --noheadings --units=B --nosuffix -o pv_free %s' % self.vdevs[0])
        self.assertEqual(pv_size, pv_free)

        # remove the old pv from the vg
        vg.RemoveDevice(old_pv, False, self.no_options, dbus_interface=self.iface_prefix + '.VolumeGroup')

        _ret, out = self.run_command('pvs --noheadings -o vg_name %s' % self.vdevs[0])
        self.assertEqual(out, '')

class UdisksLVMVDOTest(UDisksLVMTestBase):
    '''This is a basic LVM VDO test suite'''

    LOOP_DEVICE_PATH = '/var/tmp/udisks_test_disk_lvmvdo'

    @classmethod
    def setUpClass(cls):
        UDisksLVMTestBase.setUpClass()

        if not cls.module_available("kvdo"):
            udiskstestcase.UdisksTestCase.tearDownClass()
            raise unittest.SkipTest('VDO kernel module not available, skipping.')

        lvm_version = cls._get_lvm_version()
        if lvm_version < Version('2.3.07'):
            udiskstestcase.UdisksTestCase.tearDownClass()
            raise unittest.SkipTest('LVM >= 2.3.07 is needed for LVM VDO, skipping.')

    def setUp(self):
        # create backing sparse file
        # VDO needs at least 5G of space and we need some room for the grow test
        # ...rumors go that vdo internally operates on 2G extents...
        self.run_command('truncate -s 8G %s' % self.LOOP_DEVICE_PATH)
        ret_code, self.dev_name = self.run_command('losetup --find --show %s' % self.LOOP_DEVICE_PATH)
        self.assertEqual(ret_code, 0)
        time.sleep(0.5)
        self.device = self.get_device(self.dev_name)
        self.assertIsNotNone(self.device)
        super(UdisksLVMVDOTest, self).setUp()

    def tearDown(self):
        # need to process scheduled cleanup before the backing device is torn down
        self.doCleanups()
        # tear down loop device
        self.run_command('losetup --detach %s' % self.dev_name)
        os.remove(self.LOOP_DEVICE_PATH)
        super(UdisksLVMVDOTest, self).tearDown()

    def test_create(self):
        vgname = 'udisks_test_vdo_vg'

        # create vg on our testing device
        vg = self._create_vg(vgname, [self.device])
        self.addCleanup(self._remove_vg, vg)

        vg_free = self.get_property(vg, '.VolumeGroup', 'FreeSize')
        vg_free.assertGreater(0)
        lv_name = 'udisks_test_vdovlv'
        pool_name = 'udisks_test_vdopool'
        psize = vg_free.value
        vsize = psize * 5
        lv_path = vg.CreateVDOVolume(lv_name, pool_name, dbus.UInt64(psize), dbus.UInt64(vsize),
                                     dbus.UInt64(0), True, True, "auto", self.no_options,
                                     dbus_interface=self.iface_prefix + '.VolumeGroup')
        self.assertIsNotNone(lv_path)

        ret, _out = self.run_command('lvs %s' % os.path.join(vgname, lv_name))
        self.assertEqual(ret, 0)

        lv = self.bus.get_object(self.iface_prefix, lv_path)
        self.assertIsNotNone(lv)
        self.assertHasIface(lv, self.iface_prefix + '.VDOVolume')

        dbus_name = self.get_property(lv, '.LogicalVolume', 'Name')
        dbus_name.assertEqual(lv_name)

        # lv size -> original 'virtual' size
        dbus_size = self.get_property(lv, '.LogicalVolume', 'Size')
        dbus_size.assertEqual(vsize)

        # VDO pool properties
        pool_path = self.get_property(lv, '.VDOVolume', 'VDOPool')
        pool_path.assertNotEqual('/')
        pool = self.bus.get_object(self.iface_prefix, pool_path.value)
        self.assertIsNotNone(pool)

        dbus_name = self.get_property(pool, '.LogicalVolume', 'Name')
        dbus_name.assertEqual(pool_name)

        # pool size -> original 'physical' size
        dbus_size = self.get_property(pool, '.LogicalVolume', 'Size')
        dbus_size.assertEqual(psize)

        dbus_type = self.get_property(lv, '.LogicalVolume', 'Type')
        dbus_type.assertNotEqual('vdopool')

        # VDO properties
        dbus_comp = self.get_property(lv, '.VDOVolume', 'Compression')
        dbus_comp.assertTrue()

        dbus_dedup = self.get_property(lv, '.VDOVolume', 'Deduplication')
        dbus_dedup.assertTrue()

        # ThinPool property should not be set
        dbus_tp = self.get_property(lv, '.LogicalVolume', 'ThinPool')
        dbus_tp.assertEqual('/')

        # get statistics and do some simple sanity check
        stats = lv.GetStatistics(self.no_options, dbus_interface=self.iface_prefix + '.VDOVolume')
        self.assertIn("writeAmplificationRatio", stats.keys())

    def test_enable_disable_compression_deduplication(self):
        vgname = 'udisks_test_vdo_vg'

        # create vg on our testing device
        vg = self._create_vg(vgname, [self.device])
        self.addCleanup(self._remove_vg, vg)

        vg_free = self.get_property(vg, '.VolumeGroup', 'FreeSize')
        vg_free.assertGreater(0)
        lv_name = 'udisks_test_vdovlv'
        pool_name = 'udisks_test_vdopool'
        psize = vg_free.value
        vsize = psize * 5
        lv_path = vg.CreateVDOVolume(lv_name, pool_name, dbus.UInt64(psize), dbus.UInt64(vsize),
                                     dbus.UInt64(0), True, True, "auto", self.no_options,
                                     dbus_interface=self.iface_prefix + '.VolumeGroup')
        self.assertIsNotNone(lv_path)

        lv = self.bus.get_object(self.iface_prefix, lv_path)
        self.assertIsNotNone(lv)

        # initial state: both compression and deduplication should be enabled
        dbus_comp = self.get_property(lv, '.VDOVolume', 'Compression')
        dbus_comp.assertTrue()

        dbus_dedup = self.get_property(lv, '.VDOVolume', 'Deduplication')
        dbus_dedup.assertTrue()

        # disable deduplication
        lv.EnableDeduplication(False, self.no_options,
                               dbus_interface=self.iface_prefix + '.VDOVolume')
        dbus_dedup = self.get_property(lv, '.VDOVolume', 'Deduplication')
        dbus_dedup.assertFalse()

        # disable compression
        lv.EnableCompression(False, self.no_options,
                             dbus_interface=self.iface_prefix + '.VDOVolume')
        dbus_comp = self.get_property(lv, '.VDOVolume', 'Compression')
        dbus_comp.assertFalse()

        # enable both again
        lv.EnableDeduplication(True, self.no_options,
                               dbus_interface=self.iface_prefix + '.VDOVolume')
        dbus_dedup = self.get_property(lv, '.VDOVolume', 'Deduplication')
        dbus_dedup.assertTrue()

        # disable compression
        lv.EnableCompression(True, self.no_options,
                             dbus_interface=self.iface_prefix + '.VDOVolume')
        dbus_comp = self.get_property(lv, '.VDOVolume', 'Compression')
        dbus_comp.assertTrue()

    def test_resize_logical(self):
        vgname = 'udisks_test_vdo_vg'

        # create vg on our testing device
        vg = self._create_vg(vgname, [self.device])
        self.addCleanup(self._remove_vg, vg)

        vg_free = self.get_property(vg, '.VolumeGroup', 'FreeSize')
        vg_free.assertGreater(0)
        lv_name = 'udisks_test_vdovlv'
        pool_name = 'udisks_test_vdopool'
        psize = vg_free.value
        vsize = psize * 2
        lv_path = vg.CreateVDOVolume(lv_name, pool_name, dbus.UInt64(psize), dbus.UInt64(vsize),
                                     dbus.UInt64(0), True, True, "auto", self.no_options,
                                     dbus_interface=self.iface_prefix + '.VolumeGroup')
        self.assertIsNotNone(lv_path)

        lv = self.bus.get_object(self.iface_prefix, lv_path)
        self.assertIsNotNone(lv)

        lv.ResizeLogical(vsize * 5, self.no_options,
                         dbus_interface=self.iface_prefix + '.VDOVolume')
        dbus_size = self.get_property(lv, '.LogicalVolume', 'Size')
        dbus_size.assertEqual(vsize * 5)

    @udiskstestcase.tag_test(udiskstestcase.TestTags.UNSTABLE)
    def test_resize_physical(self):
        vgname = 'udisks_test_vdo_vg'

        # create vg on our testing device
        vg = self._create_vg(vgname, [self.device])
        self.addCleanup(self._remove_vg, vg)

        vg_free = self.get_property(vg, '.VolumeGroup', 'FreeSize')
        vg_free.assertGreater(2 * 1024**3)
        lv_name = 'udisks_test_vdovlv'
        pool_name = 'udisks_test_vdopool'
        psize = vg_free.value - 2 * 1024**3
        vsize = psize * 5
        lv_path = vg.CreateVDOVolume(lv_name, pool_name, dbus.UInt64(psize), dbus.UInt64(vsize),
                                     dbus.UInt64(0), True, True, "auto", self.no_options,
                                     dbus_interface=self.iface_prefix + '.VolumeGroup')
        self.assertIsNotNone(lv_path)

        lv = self.bus.get_object(self.iface_prefix, lv_path)
        self.assertIsNotNone(lv)

        lv.ResizePhysical(vg_free.value, self.no_options,
                          dbus_interface=self.iface_prefix + '.VDOVolume')

        pool_path = self.get_property(lv, '.VDOVolume', 'VDOPool')
        pool_path.assertNotEqual('/')
        pool = self.bus.get_object(self.iface_prefix, pool_path.value)
        self.assertIsNotNone(pool)

        dbus_size = self.get_property(pool, '.LogicalVolume', 'Size')
        dbus_size.assertEqual(vg_free.value)

class UdisksLVMTeardownTest(UDisksLVMTestBase):
    '''Stacked LVM + LUKS automatic teardown tests'''

    PASSPHRASE = 'einszweidrei'

    def setUp(self):
        super(UdisksLVMTeardownTest, self).setUp()

    def tearDown(self):
        self.doCleanups()
        super(UdisksLVMTeardownTest, self).tearDown()

    def _remove_luks(self, device, name, close=True):
        if close:
            try:
                self.remove_file('/etc/luks-keys/%s' % name, ignore_nonexistent=True)
                device.Lock(self.no_options, dbus_interface=self.iface_prefix + '.Encrypted')
            except dbus.exceptions.DBusException as e:
                # ignore when luks is actually already locked
                if not str(e).endswith('is not unlocked') and not 'No such interface' in str(e) and \
                   not 'Object does not exist at path' in str(e):
                    raise e

        try:
            d = dbus.Dictionary(signature='sv')
            d['erase'] = True
            device.Format('empty', d, dbus_interface=self.iface_prefix + '.Block')
        except dbus.exceptions.DBusException as e:
            if not 'No such interface' in str(e) and not 'Object does not exist at path' in str(e):
                raise e

    def _init_stack(self, name):
        vgname = name + '_vg'
        lvname = name + '_lv'

        # backup and restore
        crypttab = self.read_file('/etc/crypttab')
        self.addCleanup(self.write_file, '/etc/crypttab', crypttab)
        fstab = self.read_file('/etc/fstab')
        self.addCleanup(self.write_file, '/etc/fstab', fstab)

        # create VG with one PV
        self.pv = self.get_object('/block_devices/' + os.path.basename(self.vdevs[0]))
        self.assertIsNotNone(self.pv)

        self.vg = self._create_vg(vgname, dbus.Array([self.pv]))
        self.vg_path = self.vg.object_path
        self.addCleanup(self._remove_vg, self.vg, tear_down=True, ignore_removed=True)

        # create an LV on it
        self.lv_path = self.vg.CreatePlainVolume(lvname, dbus.UInt64(200 * 1024**2), self.no_options,
                                                 dbus_interface=self.iface_prefix + '.VolumeGroup')
        self.lv = self.bus.get_object(self.iface_prefix, self.lv_path)
        self.assertIsNotNone(self.lv)

        self.lv_block_path = self.lv.Activate(self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')
        self.assertIsNotNone(self.lv_block_path)

        self.lv_block = self.get_object(self.lv_block_path)
        self.assertIsNotNone(self.lv_block)

        # create LUKS on the LV
        options = dbus.Dictionary(signature='sv')
        options['encrypt.type'] = 'luks2'
        options['encrypt.passphrase'] = self.PASSPHRASE
        options['label'] = 'COCKPITFS'
        options['tear-down'] = dbus.Boolean(True)

        crypttab_items = dbus.Dictionary({'name': self.str_to_ay(vgname),
                                          'options': self.str_to_ay('verify,discard'),
                                          'passphrase-contents': self.str_to_ay(self.PASSPHRASE),
                                          'track-parents': True},
                                          signature=dbus.Signature('sv'))
        fstab_items = dbus.Dictionary({'dir': self.str_to_ay(vgname),
                                       'type': self.str_to_ay('ext4'),
                                       'opts': self.str_to_ay('defaults'),
                                       'freq': 0, 'passno': 0,
                                       'track-parents': True},
                                      signature=dbus.Signature('sv'))
        options['config-items'] = dbus.Array([('crypttab', crypttab_items),
                                              ('fstab', fstab_items)])

        self.lv_block.Format('ext4', options, dbus_interface=self.iface_prefix + '.Block')
        self.addCleanup(self._remove_luks, self.lv_block, vgname)
        self.luks_uuid = self.get_property_raw(self.lv_block, '.Block', 'IdUUID')

        self.luks_block_path = self.get_property_raw(self.lv_block, '.Encrypted', 'CleartextDevice')
        luks_block = self.get_object(self.luks_block_path)
        self.assertIsNotNone(luks_block)
        self.fs_uuid = self.get_property_raw(luks_block, '.Block', 'IdUUID')

        # check for present crypttab configuration item
        conf = self.get_property(self.lv_block, '.Block', 'Configuration')
        conf.assertTrue()
        self.assertEqual(conf.value[0][0], 'crypttab')

        # check for present fstab configuration item on a cleartext block device
        conf = self.get_property(luks_block, '.Block', 'Configuration')
        conf.assertTrue()
        self.assertEqual(conf.value[0][0], 'fstab')

        child_conf = self.get_property(self.lv_block, '.Encrypted', 'ChildConfiguration')
        child_conf.assertTrue()
        self.assertEqual(child_conf.value[0][0], 'fstab')
        self.assertEqual(child_conf.value, conf.value)

        # check that fstab and crypttab records have been added
        crypttab = self.read_file('/etc/crypttab')
        self.assertIn(vgname, crypttab)
        self.assertIn(self.luks_uuid, crypttab)
        fstab = self.read_file('/etc/fstab')
        self.assertIn(vgname, fstab)
        self.assertIn(self.fs_uuid, fstab)

    def _check_torn_down_stack(self, name):
        # check that all created objects don't exist anymore
        msg = r'Object does not exist at path|No such interface'
        with six.assertRaisesRegex(self, dbus.exceptions.DBusException, msg):
            luks_block = self.get_object(self.luks_block_path)
            self.get_property_raw(luks_block, '.Block', 'DeviceNumber')
        with six.assertRaisesRegex(self, dbus.exceptions.DBusException, msg):
            lv_block = self.get_object(self.lv_block_path)
            self.get_property_raw(lv_block, '.Block', 'DeviceNumber')
        with six.assertRaisesRegex(self, dbus.exceptions.DBusException, msg):
            # the lvm2 udisks module is not fully synchronous, see https://github.com/storaged-project/udisks/pull/814
            time.sleep(2)
            lv = self.get_object(self.lv_path)
            self.get_property_raw(lv, '.LogicalVolume', 'Name')
        with six.assertRaisesRegex(self, dbus.exceptions.DBusException, msg):
            vg = self.get_object(self.vg_path)
            self.get_property_raw(vg, '.VolumeGroup', 'Name')

        # check that fstab and crypttab records have been removed
        crypttab = self.read_file('/etc/crypttab')
        self.assertNotIn(name, crypttab)
        self.assertNotIn(self.luks_uuid, crypttab)
        fstab = self.read_file('/etc/fstab')
        self.assertNotIn(name, fstab)
        self.assertNotIn(self.fs_uuid, fstab)

    @udiskstestcase.tag_test(udiskstestcase.TestTags.UNSAFE)
    def test_teardown_active_vg_unlocked(self):
        ''' Test tear-down by removing the base VG (not deactivated, unlocked) '''

        name = 'udisks_test_teardown_active_vg_unlocked'

        self._init_stack(name)

        self._remove_vg(self.vg, tear_down=True, ignore_removed=False)

        self._check_torn_down_stack(name)

    @udiskstestcase.tag_test(udiskstestcase.TestTags.UNSAFE)
    def test_teardown_active_vg_locked(self):
        ''' Test tear-down by removing the base VG (not deactivated, locked) '''

        name = 'udisks_test_teardown_active_vg_locked'

        self._init_stack(name)

        self.lv_block.Lock(self.no_options, dbus_interface=self.iface_prefix + '.Encrypted')
        self._remove_vg(self.vg, tear_down=True, ignore_removed=False)

        self._check_torn_down_stack(name)

    @udiskstestcase.tag_test(udiskstestcase.TestTags.UNSAFE)
    def test_teardown_inactive_vg_locked(self):
        ''' Test tear-down by removing the base VG (deactivated, locked) '''

        name = 'udisks_test_teardown_inactive_locked'

        self._init_stack(name)

        self.lv_block.Lock(self.no_options, dbus_interface=self.iface_prefix + '.Encrypted')
        self.lv.Deactivate(self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')
        self._remove_vg(self.vg, tear_down=True, ignore_removed=False)

        self._check_torn_down_stack(name)

    @udiskstestcase.tag_test(udiskstestcase.TestTags.UNSAFE)
    def test_reformat_inactive_vg_locked(self):
        ''' Test tear-down by re-formatting the base PV (VG deactivated, locked) '''

        name = 'test_reformat_inactive_vg_locked'

        self._init_stack(name)

        self.lv_block.Lock(self.no_options, dbus_interface=self.iface_prefix + '.Encrypted')
        self.lv.Deactivate(self.no_options, dbus_interface=self.iface_prefix + '.LogicalVolume')

        # now reformat the PV with tear-down flag
        options = dbus.Dictionary(signature='sv')
        options['label'] = 'AFTER_TEARDOWN'
        options['tear-down'] = dbus.Boolean(True)

        self.pv.Format('ext4', options, dbus_interface=self.iface_prefix + '.Block')
        self.addCleanup(self.wipe_fs, self.vdevs[0])

        # TODO: implement proper teardown across combined LVM + LUKS stack
        # https://github.com/storaged-project/udisks/issues/781

        # check that fstab and crypttab records have been removed
        # TODO: these checks are the opposite - record shouldn't be present, once this is fixed
        # self._check_torn_down_stack(name)
        crypttab = self.read_file('/etc/crypttab')
        self.assertIn(name, crypttab)
        self.assertIn(self.luks_uuid, crypttab)
        fstab = self.read_file('/etc/fstab')
        self.assertIn(name, fstab)
        self.assertIn(self.fs_uuid, fstab)


import requests
import json
import time
from pygame import mixer
#Enter your url here and enjoy your free offer!!
url = 'https://www.cricbuzz.com/match-api/21859/commentary.json'

f = 0
c = time.time()
while True:
	try:
		current_scorecard = requests.get(url).json()['score']['prev_overs']
	except:
		mixer.init()
		mixer.music.load('error.mp3')
		mixer.music.play()
		break
	for score in current_scorecard[-10:-1]:
		if score == '6':
			mixer.init()
			mixer.music.load('six.mp3')
			mixer.music.play()
			f = 1
			break
	if f == 1:
		break
	time.sleep(5*60)


from django.shortcuts import render
from patient.models import Patient
from .models import Need, Appointment
from .forms import AddNeedForm

def add_need(request, patient_number):
    form = AddNeedForm(request.POST or None)
    current_patient = Patient.objects.get(id=patient_number)
    if form.is_valid():
        need = Need(need_string=form.cleaned_data['need_string'],
                    date=form.cleaned_data['date'],
                    start=form.cleaned_data['start'],
                    end=form.cleaned_data['end'],
                    duration_heal=form.cleaned_data['duration_heal'],
                    patient=current_patient)
        need.save()
        success = True
    return render(request, 'need/add_need.html', locals())

def appointment_detail(request, id_appointment):
    try:
        appointment = Appointment.objects.get(id=id_appointment)
        patient = appointment.need.patient
    except Appointment.DoesNotExist:
        exception_raised = True
    return render(request, 'event/appointment_detail.html', locals())


#!/usr/bin/env python3

import argparse
import os
import unittest
import testtools
import sys

PROJECT_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.path.pardir)
sys.path.append(PROJECT_DIR)

parser = argparse.ArgumentParser(description="Run tests.")
parser.add_argument("--deployment", choices=["aws", "azure", "local"], nargs="+")

args = parser.parse_args()
if not args.deployment:
    args.deployment = []

# https://stackoverflow.com/questions/22484805/a-simple-working-example-for-testtools-concurrentstreamtestsuite
class TracingStreamResult(testtools.StreamResult):
    all_correct: bool
    output = {}

    def __init__(self):
        self.all_correct = True

    def status(self, *args, **kwargs):
        self.all_correct = self.all_correct and (kwargs["test_status"] in ["inprogress", "success"])
        if not kwargs["test_status"]:
            test_id = kwargs["test_id"]
            if test_id not in self.output:
                self.output[test_id] = b""
            self.output[test_id] += kwargs["file_bytes"]
        elif kwargs["test_status"] == "fail":
            print('{0[test_id]}: {0[test_status]}'.format(kwargs))
            print('{0[test_id]}: {1}'.format(kwargs, self.output[kwargs["test_id"]].decode()))
        elif kwargs["test_status"] == "success":
            print('{0[test_id]}: {0[test_status]}'.format(kwargs))

cases = []
if "aws" in args.deployment:
    from aws import suite
    for case in suite.suite():
        cases.append(case)
tests = []
for case in cases:
    for c in case:
        tests.append(c)
for test in tests:
    test.setUpClass()
concurrent_suite = testtools.ConcurrentStreamTestSuite(lambda: ((test, None) for test in tests))
result = TracingStreamResult()
result.startTestRun()
concurrent_suite.run(result)
result.stopTestRun()
for test in tests:
    test.tearDownClass()
sys.exit(not result.all_correct)


from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import *
import logging
import pymongo
import pickle
import numpy as np
import bson
import geojson as gj
# This change should be removed in the next server update, by which time hopefully the new geojson version will incorporate the long-term fix for their default precision
# See - jazzband/geojson#177
# See = https://github.com/e-mission/e-mission-server/pull/900/commits/d2ada640f260aad8cbcfecb81345f4087c810baa
gj.geometry.Geometry.__init__.__defaults__ = (None, False, 15)

import emission.core.wrapper.common_trip as ecwct
import emission.core.get_database as edb
import emission.storage.decorations.trip_queries as esdtq

# constants
DAYS_IN_WEEK = 7
HOURS_IN_DAY = 24

#################################################################################
############################ database functions #################################
#################################################################################

def save_common_trip(common_trip):
    db = edb.get_common_trip_db()
    probs = _np_array_to_json_format(common_trip.probabilites)
    db.insert({
        "user_id" : common_trip.user_id,
        "start_place" : common_trip.start_place,
        "end_place" : common_trip.end_place,
        "start_loc" : common_trip.start_loc,
        "end_loc" : common_trip.end_loc,
        "trips" : common_trip["trips"],
        "probabilites" : probs,
        "start_times": common_trip["start_times"],
        "durations": common_trip["durations"]
        })

def get_common_trip_from_db(user_id, start_place_id, end_place_id):
    db = edb.get_common_trip_db()
    json_obj = db.find_one({"user_id" : user_id,
        "start_place" : start_place_id,
        "end_place" : end_place_id})
    return make_common_trip_from_json(json_obj)

def get_all_common_trips_for_user(user_id):
    db = edb.get_common_trip_db()
    return db.find({"user_id" : user_id})

def make_common_trip_from_json(json_obj):
    probs = _json_to_np_array(json_obj["probabilites"])
    props = {
        "user_id" : json_obj["user_id"],
        "start_place" : json_obj["start_place"],
        "end_place" : json_obj["end_place"],
        "start_loc" : json_obj["start_loc"],
        "end_loc" : json_obj["end_loc"],
        "trips" : json_obj["trips"],
        "probabilites" : probs
    }
    return ecwct.CommonTrip(props)

def _np_array_to_json_format(array):
    return array.tolist()

def _json_to_np_array(mongo_thing):
    return np.array(mongo_thing)

def make_new_common_trip(props=None):
    if props:
        return ecwct.CommonTrip(props)
    return ecwct.CommonTrip()

def clear_existing_trips(user_id):
    db = edb.get_common_trip_db()
    db.remove({'user_id': user_id})

##############################################################################

def get_weight(common_trip):
    return len(common_trip["trips"])

def add_real_trip_id(trip, _id):
    trip.trips.append(_id)

def get_start_hour(section_info):
    return section_info.data.start_local_dt.hour

def get_day(section_info):
    return section_info.data.start_local_dt.weekday

def get_start_time(section_info):
    return section_info.data.start_local_dt

def increment_probability(trip, day, hour):
    trip.probabilites[day, hour] += 1

def set_up_trips(list_of_cluster_data, user_id):
    # Import in here to avoid recursive imports
    # TODO: This should really be moved to a separate class that creates the
    # entire graph at one time
    import emission.storage.decorations.common_place_queries as esdcpq
    clear_existing_trips(user_id)
    for dct in list_of_cluster_data:
        start_times = []
        durations = []
        start_loc = gj.Point(dct['start_coords'])
        end_loc = gj.Point(dct['end_coords'])
        start_place_id = esdcpq.get_common_place_at_location(start_loc).get_id()
        end_place_id = esdcpq.get_common_place_at_location(end_loc).get_id()
        #print 'dct["sections"].trip_id %s is' % dct["sections"][0]
        probabilites = np.zeros((DAYS_IN_WEEK, HOURS_IN_DAY))
        for sec in dct["sections"]:
            probabilites[get_day(sec), get_start_hour(sec)] += 1
            start_times.append(get_start_time(sec))
            durations.append(sec.data.duration)

        trip = make_new_common_trip()
        trip.user_id = user_id
        trip.start_place = start_place_id
        trip.end_place = end_place_id
        trip.start_loc = start_loc
        trip.end_loc = end_loc
        trip.probabilites = probabilites
        trip.trips = [unc_trip.get_id() for unc_trip in dct["sections"]]
        trip.start_times = start_times
        trip.durations = durations

        save_common_trip(trip)


import time
import asyncio
import tornado.web
import tornado.ioloop

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.write("Hello, world\n")

class NonBlocking(tornado.web.RequestHandler):
    async def get(self):
        await asyncio.sleep(10)

class Blocking(tornado.web.RequestHandler):
    def get(self):
        time.sleep(10)

def make_app():
    return tornado.web.Application([
        (r"/", MainHandler),
        (r"/non_blocking", NonBlocking),
        (r"/blocking", Blocking),
    ])

if __name__ == "__main__":
    app = make_app()
    app.listen(8888)
    tornado.ioloop.IOLoop.current().start()


import pandas as pd
import datetime
import json

def date_to_week(d):
    split_date = d.split('/')
    return int(datetime.date(int(split_date[2]), int(split_date[0]), int(split_date[1])).strftime('%U'))

# Historical deaths data (and 2020, which will be overwritten where possible)
deaths = pd.read_csv('deaths_2020-06-24.csv')
deaths = deaths.drop(columns=['Type', 'Week Ending Date', 'Time Period', 'Suppress','Note','State Abbreviation'])

deaths = deaths.rename(columns={'Age Group': 'Slice'})
deaths = deaths.rename(columns={'Number of Deaths': 'Deaths'})

#deaths = deaths.set_index(['Jurisdiction', 'Year', 'Slice', 'Week'])

all_deaths = deaths.set_index(['Jurisdiction', 'Year', 'Slice', 'Week']).sum(level=[0, 1, 3]).reset_index()
all_deaths['Slice'] = 'All Deaths'

mean_deaths = all_deaths[all_deaths['Year'] != 2020].set_index(['Jurisdiction', 'Year', 'Slice', 'Week']).mean(level=[0, 2, 3]).reset_index()
mean_deaths['Year'] = '2015-2019 Mean'

max_deaths = all_deaths[all_deaths['Year'] != 2020].set_index(['Jurisdiction', 'Year', 'Slice', 'Week']).max(level=[0, 2, 3]).reset_index()
max_deaths['Year'] = '2015-2019 Max'

# 2020 total deaths, and COVID deaths (filtering out other "select deaths numbers")
covid_deaths = pd.read_csv('covid19_deaths_2020-06-25.csv')
covid_deaths = covid_deaths.drop(columns=['Influenza Deaths', 'Pneumonia and COVID-19 Deaths', 'Pneumonia Deaths'])
covid_deaths = covid_deaths.drop(columns=['Percent of Expected Deaths', 'Data as of', 'Start week', 'Group'])
covid_deaths = covid_deaths.drop(columns=['Pneumonia, Influenza, or COVID-19 Deaths', 'Indicator', 'Footnote'])

covid_deaths = covid_deaths.rename(columns={'State': 'Jurisdiction'})
covid_deaths = covid_deaths.rename(columns={'End Week': 'Week'})
covid_deaths = covid_deaths.rename(columns={'Total Deaths': 'All Deaths'})

covid_deaths['Week'] = covid_deaths['Week'].map(date_to_week)
covid_deaths['Year'] = 2020

covid_deaths = covid_deaths.melt(var_name='Slice', value_vars=['COVID-19 Deaths', 'All Deaths'], value_name='Deaths', id_vars=['Jurisdiction', 'Year', 'Week'])
#covid_deaths = covid_deaths.set_index(['Jurisdiction', 'Year', 'Slice', 'Week'])

covid_mean_implied = pd.concat([covid_deaths[covid_deaths['Slice'] == 'COVID-19 Deaths'], mean_deaths])\
    .set_index(['Jurisdiction', 'Year', 'Slice', 'Week']).sum(level=[0, 3]).reset_index()
covid_mean_implied['Year'] = '2015-2019 Mean + COVID-19 Reported'
covid_mean_implied['Slice'] = 'All Deaths'

covid_max_implied = pd.concat([covid_deaths[covid_deaths['Slice'] == 'COVID-19 Deaths'], max_deaths])\
    .set_index(['Jurisdiction', 'Year', 'Slice', 'Week']).sum(level=[0, 3]).reset_index()
covid_max_implied['Year'] = '2015-2019 Max + COVID-19 Reported'
covid_max_implied['Slice'] = 'All Deaths'

deaths_all = pd.concat([deaths, all_deaths, mean_deaths, max_deaths, covid_mean_implied, covid_max_implied, covid_deaths])
deaths_all = deaths_all.drop_duplicates(subset=['Jurisdiction', 'Year', 'Slice', 'Week'], keep='last') # Take more recent covid deaths total data over older historical dataset (which includes some 2020)
#deaths_all = deaths_all.set_index(['Jurisdiction', 'Year', 'Slice', 'Week'])

deaths_all = deaths_all.pivot_table(index='Week', columns=['Jurisdiction', 'Year', 'Slice'], values='Deaths')

# Cannot find a good pandas way to do this
deaths_dict = {}
for index in deaths_all:
    # deaths_dict[index] = deaths_all[index]
    traverse = deaths_dict
    for i in range(len(index) - 1):
        if index[i] not in traverse:
            traverse[index[i]] = {}

        traverse = traverse[index[i]]

    traverse[index[-1]] = deaths_all[index].dropna().to_dict()

    #print(deaths)

with open('src/deaths.json', 'w') as fp:
    json.dump(deaths_dict, fp, allow_nan=False)


import getpass
import os
import subprocess

from paramiko.client import SSHClient, AutoAddPolicy
from paramiko.config import SSHConfig

from .transfer import Transfer

class Connection(object):

    host = None
    original_host = None
    user = None
    port = None
    ssh_config = None
    connect_timeout = None
    connect_kwargs = None
    client = None
    transport = None
    _sftp = None

    def __init__(
        self,
        host,
        user=None,
        port=None,
        connect_timeout=None,
        connect_kwargs=None,
    ):
        shorthand = self.derive_shorthand(host)
        host = shorthand['host']
        err = (
            'You supplied the {} via both shorthand and kwarg! Please pick one.'  # noqa
        )
        if shorthand['user'] is not None:
            if user is not None:
                raise ValueError(err.format('user'))
            user = shorthand['user']
        if shorthand['port'] is not None:
            if port is not None:
                raise ValueError(err.format('port'))
            port = shorthand['port']

        self.ssh_config = self.load_ssh_config(host)

        self.original_host = host
        self.host = host
        if 'hostname' in self.ssh_config:
            self.host = self.ssh_config['hostname']

        self.user = user or self.ssh_config.get('user', getpass.getuser())
        self.port = port or int(self.ssh_config.get('port', '22'))

        if connect_timeout is None:
            connect_timeout = self.ssh_config.get('connecttimeout')
        if connect_timeout is not None:
            connect_timeout = int(connect_timeout)
        self.connect_timeout = connect_timeout

        if connect_kwargs is None:
            connect_kwargs = {}
        if 'identityfile' in self.ssh_config:
            connect_kwargs.setdefault('key_filename', [])
            connect_kwargs['key_filename'].extend(
                self.ssh_config['identityfile']
            )
        self.connect_kwargs = connect_kwargs

        client = SSHClient()
        client.set_missing_host_key_policy(AutoAddPolicy())
        self.client = client

        self.transport = None

    def derive_shorthand(self, host_string):
        user_hostport = host_string.rsplit('@', 1)
        hostport = user_hostport.pop()
        user = user_hostport[0] if user_hostport and user_hostport[0] else None

        # IPv6: can't reliably tell where addr ends and port begins, so don't
        # try (and don't bother adding special syntax either, user should avoid
        # this situation by using port=).
        if hostport.count(':') > 1:
            host = hostport
            port = None
        # IPv4: can split on ':' reliably.
        else:
            host_port = hostport.rsplit(':', 1)
            host = host_port.pop(0) or None
            port = host_port[0] if host_port and host_port[0] else None

        if port is not None:
            port = int(port)

        return {'user': user, 'host': host, 'port': port}

    def load_ssh_config(self, host):
        ssh_config = SSHConfig()

        for path in (
            os.path.expanduser('~/.ssh/config'),
            '/etc/ssh/ssh_config'
        ):
            if os.path.isfile(path):
                with open(path) as fd:
                    ssh_config.parse(fd)

        return ssh_config.lookup(host)

    @property
    def is_connected(self):
        return self.transport.active if self.transport else False

    def open(self):
        if self.is_connected:
            return
        err = (
            "Refusing to be ambiguous: connect() kwarg '{}' was given both via regular arg and via connect_kwargs!"  # noqa
        )
        for key in 'hostname port username'.split():
            if key in self.connect_kwargs:
                raise ValueError(err.format(key))
        if (
            'timeout' in self.connect_kwargs
            and self.connect_timeout is not None
        ):
            raise ValueError(err.format('timeout'))
        # No conflicts -> merge 'em together
        kwargs = dict(
            self.connect_kwargs,
            username=self.user,
            hostname=self.host,
            port=self.port,
        )
        if self.connect_timeout:
            kwargs['timeout'] = self.connect_timeout
        if 'key_filename' in kwargs and not kwargs['key_filename']:
            del kwargs['key_filename']

        self.client.connect(**kwargs)
        self.transport = self.client.get_transport()

    def close(self):
        if self.is_connected:
            self.client.close()

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        self.close()

    def run(self, command, **kwargs):
        self.open()
        return self.client.exec_command(command, **kwargs)

    def sudo(self, command, **kwargs):
        self.open()
        return self.client.exec_command('sudo -S -p ' + command, **kwargs)

    def local(self, command, **kwargs):
        return subprocess.call(command, shell=True, **kwargs)

    def sftp(self):
        self.open()
        if self._sftp is None:
            self._sftp = self.client.open_sftp()
        return self._sftp

    def get(self, *args, **kwargs):
        return Transfer(self).get(*args, **kwargs)

    def put(self, *args, **kwargs):
        return Transfer(self).put(*args, **kwargs)


#!/bin/env python3

import sys
import re
from functools import cache

def readfile(filename):
  with open(filename) as f:
    inp = f.read()
  return inp

def step1(inp):
  return recurse(inp, findwire(inp, 'a'))

def step2(inp, override):
  inp = re.sub('.*-> b\n', f'{override} -> b\n', inp)
  return step1(inp)

def findwire(inp, wire):
  return re.search(f'(.*-> {wire})\n', inp).group(1).split(' -> ')[0]

@cache
def recurse(inp, instr):
  expr = ''

  for i in instr.split(' '):
    if str(i).isalpha():
      i = recurse(inp, findwire(inp, i))
    expr += str(i)

  return eval(expr)

def main():
  filename = "input.txt"

  if len(sys.argv) > 1:
    filename = sys.argv[1]
  inp = readfile(filename)

  inp = inp.replace("AND", "&").replace("OR", "|").replace("LSHIFT", "<<").replace("RSHIFT", ">>").replace("NOT", "65535 -")

  step1_out = step1(inp)
  step2_out = step2(inp, step1_out)

  print(f'Step 1: {step1_out}')
  print(f'Step 2: {step2_out}')

if __name__ == '__main__':
  main()

##!/bin/env python3
#
#import re
#
#with open('input.txt') as f:
#    lines = f.read().splitlines()
#
#step1 = 0
#step2 = 0
#
#signals = {}
#
#for l in lines:
#    inp, out = l.split(" -> ")
#
#    inp = re.sub(r"([a-z]+)", r"signals['\1']", inp)
#
#    inp = inp.replace("AND", "&").replace("OR", "|").replace("LSHIFT", "<<").replace("RSHIFT", ">>").replace("NOT", "65535 -")
#    print(F"{out} = {inp}")
#    signals[out] = eval(inp)
#
#print(signals['a'])
#
#
#
#
#print(F"Step 1: {step1}")
#print(F"Step 2: {step2}")


from __future__ import unicode_literals

"""Benchmark for SQLAlchemy.

An adaptation of Robert Brewers' ZooMark speed tests. """

import datetime
from sqlalchemy import Table, Column, Integer, Unicode, Date, \
    DateTime, Time, Float, Sequence, ForeignKey,  \
    select, join, and_, outerjoin, func
from sqlalchemy.testing import replay_fixture

ITERATIONS = 1

class ZooMarkTest(replay_fixture.ReplayFixtureTest):

    """Runs the ZooMark and squawks if method counts vary from the norm."""

    __requires__ = 'cpython',
    __only_on__ = 'postgresql+psycopg2'

    def _run_steps(self, ctx):
        with ctx():
            self._baseline_1a_populate()
        with ctx():
            self._baseline_2_insert()
        with ctx():
            self._baseline_3_properties()
        with ctx():
            self._baseline_4_expressions()
        with ctx():
            self._baseline_5_aggregates()
        with ctx():
            self._baseline_6_editing()
        with ctx():
            self._baseline_7_multiview()

    def setup_engine(self):
        self._baseline_1_create_tables()

    def teardown_engine(self):
        self._baseline_8_drop()

    def _baseline_1_create_tables(self):
        Table(
            'Zoo',
            self.metadata,
            Column('ID', Integer, Sequence('zoo_id_seq'),
                   primary_key=True, index=True),
            Column('Name', Unicode(255)),
            Column('Founded', Date),
            Column('Opens', Time),
            Column('LastEscape', DateTime),
            Column('Admission', Float),
        )
        Table(
            'Animal',
            self.metadata,
            Column('ID', Integer, Sequence('animal_id_seq'),
                   primary_key=True),
            Column('ZooID', Integer, ForeignKey('Zoo.ID'), index=True),
            Column('Name', Unicode(100)),
            Column('Species', Unicode(100)),
            Column('Legs', Integer, default=4),
            Column('LastEscape', DateTime),
            Column('Lifespan', Float(4)),
            Column('MotherID', Integer, ForeignKey('Animal.ID')),
            Column('PreferredFoodID', Integer),
            Column('AlternateFoodID', Integer),
        )
        self.metadata.create_all()

    def _baseline_1a_populate(self):
        Zoo = self.metadata.tables['Zoo']
        Animal = self.metadata.tables['Animal']
        engine = self.metadata.bind
        wap = engine.execute(Zoo.insert(), Name='Wild Animal Park',
                             Founded=datetime.date(2000, 1, 1),
                             Opens=datetime.time(8, 15, 59),
                             LastEscape=datetime.datetime(
                                 2004, 7, 29, 5, 6, 7),
                             Admission=4.95).inserted_primary_key[0]
        sdz = engine.execute(Zoo.insert(), Name='San Diego Zoo',
                             Founded=datetime.date(1935, 9, 13),
                             Opens=datetime.time(9, 0, 0),
                             Admission=0).inserted_primary_key[0]
        engine.execute(Zoo.insert(inline=True), Name='Montr\xe9al Biod\xf4me',
                       Founded=datetime.date(1992, 6, 19),
                       Opens=datetime.time(9, 0, 0), Admission=11.75)
        seaworld = engine.execute(Zoo.insert(), Name='Sea_World',
                                  Admission=60).inserted_primary_key[0]

        # Let's add a crazy futuristic Zoo to test large date values.

        engine.execute(
            Zoo.insert(), Name='Luna Park',
            Founded=datetime.date(2072, 7, 17),
            Opens=datetime.time(0, 0, 0),
            Admission=134.95).inserted_primary_key[0]

        # Animals

        leopardid = engine.execute(Animal.insert(), Species='Leopard',
                                   Lifespan=73.5).inserted_primary_key[0]
        engine.execute(Animal.update(Animal.c.ID == leopardid), ZooID=wap,
                       LastEscape=datetime.datetime(
                           2004, 12, 21, 8, 15, 0, 999907,)
                       )
        engine.execute(
            Animal.insert(),
            Species='Lion', ZooID=wap).inserted_primary_key[0]

        engine.execute(Animal.insert(), Species='Slug', Legs=1, Lifespan=.75)
        engine.execute(Animal.insert(), Species='Tiger',
                       ZooID=sdz).inserted_primary_key[0]

        # Override Legs.default with itself just to make sure it works.

        engine.execute(Animal.insert(inline=True), Species='Bear', Legs=4)
        engine.execute(Animal.insert(inline=True), Species='Ostrich', Legs=2,
                       Lifespan=103.2)
        engine.execute(Animal.insert(inline=True), Species='Centipede',
                       Legs=100)
        engine.execute(Animal.insert(), Species='Emperor Penguin',
                       Legs=2, ZooID=seaworld).inserted_primary_key[0]
        engine.execute(Animal.insert(), Species='Adelie Penguin',
                       Legs=2, ZooID=seaworld).inserted_primary_key[0]
        engine.execute(Animal.insert(inline=True), Species='Millipede',
                       Legs=1000000, ZooID=sdz)

        # Add a mother and child to test relationships

        bai_yun = engine.execute(
            Animal.insert(),
            Species='Ape',
            Name='Bai Yun',
            Legs=2).inserted_primary_key[0]
        engine.execute(Animal.insert(inline=True), Species='Ape',
                       Name='Hua Mei', Legs=2, MotherID=bai_yun)

    def _baseline_2_insert(self):
        Animal = self.metadata.tables['Animal']
        i = Animal.insert(inline=True)
        for x in range(ITERATIONS):
            i.execute(Species='Tick', Name='Tick %d' % x, Legs=8)

    def _baseline_3_properties(self):
        Zoo = self.metadata.tables['Zoo']
        Animal = self.metadata.tables['Animal']
        engine = self.metadata.bind

        def fullobject(select):
            """Iterate over the full result row."""

            return list(engine.execute(select).first())

        for x in range(ITERATIONS):

            # Zoos

            fullobject(Zoo.select(Zoo.c.Name == 'Wild Animal Park'))
            fullobject(Zoo.select(Zoo.c.Founded ==
                       datetime.date(1935, 9, 13)))
            fullobject(Zoo.select(Zoo.c.Name ==
                       'Montr\xe9al Biod\xf4me'))
            fullobject(Zoo.select(Zoo.c.Admission == float(60)))

            # Animals

            fullobject(Animal.select(Animal.c.Species == 'Leopard'))
            fullobject(Animal.select(Animal.c.Species == 'Ostrich'))
            fullobject(Animal.select(Animal.c.Legs == 1000000))
            fullobject(Animal.select(Animal.c.Species == 'Tick'))

    def _baseline_4_expressions(self):
        Zoo = self.metadata.tables['Zoo']
        Animal = self.metadata.tables['Animal']
        engine = self.metadata.bind

        def fulltable(select):
            """Iterate over the full result table."""

            return [list(row) for row in engine.execute(select).fetchall()]

        for x in range(ITERATIONS):
            assert len(fulltable(Zoo.select())) == 5
            assert len(fulltable(Animal.select())) == ITERATIONS + 12
            assert len(fulltable(Animal.select(Animal.c.Legs == 4))) \
                == 4
            assert len(fulltable(Animal.select(Animal.c.Legs == 2))) \
                == 5
            assert len(
                fulltable(
                    Animal.select(
                        and_(
                            Animal.c.Legs >= 2,
                            Animal.c.Legs < 20)))) == ITERATIONS + 9
            assert len(fulltable(Animal.select(Animal.c.Legs > 10))) \
                == 2
            assert len(fulltable(Animal.select(Animal.c.Lifespan
                                               > 70))) == 2
            assert len(fulltable(Animal.select(Animal.c.Species.
                                               startswith('L')))) == 2
            assert len(fulltable(Animal.select(Animal.c.Species.
                                               endswith('pede')))) == 2
            assert len(fulltable(Animal.select(Animal.c.LastEscape
                                               != None))) == 1
            assert len(
                fulltable(
                    Animal.select(
                        None == Animal.c.LastEscape))) == ITERATIONS + 11

            # In operator (containedby)

            assert len(fulltable(Animal.select(Animal.c.Species.like('%pede%'
                                                                     )))) == 2
            assert len(
                fulltable(
                    Animal.select(
                        Animal.c.Species.in_(
                            ['Lion', 'Tiger', 'Bear'])))) == 3

            # Try In with cell references
            class thing(object):
                pass

            pet, pet2 = thing(), thing()
            pet.Name, pet2.Name = 'Slug', 'Ostrich'
            assert len(
                fulltable(
                    Animal.select(
                        Animal.c.Species.in_([pet.Name, pet2.Name])))) == 2

            # logic and other functions

            assert len(fulltable(Animal.select(Animal.c.Species.like('Slug'
                                                                     )))) == 1
            assert len(fulltable(Animal.select(Animal.c.Species.like('%pede%'
                                                                     )))) == 2
            name = 'Lion'
            assert len(
                fulltable(
                    Animal.select(
                        func.length(
                            Animal.c.Species) == len(name)))) == ITERATIONS + 3
            assert len(
                fulltable(
                    Animal.select(
                        Animal.c.Species.like('%i%')))) == ITERATIONS + 7

            # Test now(), today(), year(), month(), day()

            assert len(
                fulltable(
                    Zoo.select(
                        and_(
                            Zoo.c.Founded != None,
                            Zoo.c.Founded < func.current_timestamp(
                                _type=Date))))) == 3
            assert len(
                fulltable(
                    Animal.select(
                        Animal.c.LastEscape == func.current_timestamp(
                            _type=Date)))) == 0
            assert len(
                fulltable(
                    Animal.select(
                        func.date_part(
                            'year',
                            Animal.c.LastEscape) == 2004))) == 1
            assert len(
                fulltable(
                    Animal.select(
                        func.date_part(
                            'month',
                            Animal.c.LastEscape) == 12))) == 1
            assert len(
                fulltable(
                    Animal.select(
                        func.date_part(
                            'day',
                            Animal.c.LastEscape) == 21))) == 1

    def _baseline_5_aggregates(self):
        Animal = self.metadata.tables['Animal']
        Zoo = self.metadata.tables['Zoo']
        engine = self.metadata.bind

        for x in range(ITERATIONS):

            # views

            view = engine.execute(select([Animal.c.Legs])).fetchall()
            legs = sorted([x[0] for x in view])
            expected = {
                'Leopard': 73.5,
                'Slug': .75,
                'Tiger': None,
                'Lion': None,
                'Bear': None,
                'Ostrich': 103.2,
                'Centipede': None,
                'Emperor Penguin': None,
                'Adelie Penguin': None,
                'Millipede': None,
                'Ape': None,
                'Tick': None,
            }
            for species, lifespan in engine.execute(
                    select([Animal.c.Species, Animal.c.Lifespan])).fetchall():
                assert lifespan == expected[species]
            expected = ['Montr\xe9al Biod\xf4me', 'Wild Animal Park']
            e = select([Zoo.c.Name],
                       and_(Zoo.c.Founded != None,
                            Zoo.c.Founded <= func.current_timestamp(),
                            Zoo.c.Founded >= datetime.date(1990,
                                                           1,
                                                           1)))
            values = [val[0] for val in engine.execute(e).fetchall()]
            assert set(values) == set(expected)

            # distinct

            legs = [
                x[0]
                for x in engine.execute(
                    select([Animal.c.Legs],
                           distinct=True)).fetchall()]
            legs.sort()

    def _baseline_6_editing(self):
        Zoo = self.metadata.tables['Zoo']
        engine = self.metadata.bind
        for x in range(ITERATIONS):

            # Edit

            SDZ = engine.execute(Zoo.select(Zoo.c.Name == 'San Diego Zoo'
                                            )).first()
            engine.execute(
                Zoo.update(
                    Zoo.c.ID == SDZ['ID']),
                Name='The San Diego Zoo',
                Founded=datetime.date(1900, 1, 1),
                Opens=datetime.time(7, 30, 0), Admission='35.00')

            # Test edits

            SDZ = engine.execute(Zoo.select(Zoo.c.Name == 'The San Diego Zoo'
                                            )).first()
            assert SDZ['Founded'] == datetime.date(1900, 1, 1), \
                SDZ['Founded']

            # Change it back

            engine.execute(Zoo.update(Zoo.c.ID == SDZ['ID'
                                                      ]), Name='San Diego Zoo',
                           Founded=datetime.date(1935, 9, 13),
                           Opens=datetime.time(9, 0, 0),
                           Admission='0')

            # Test re-edits

            SDZ = engine.execute(Zoo.select(Zoo.c.Name == 'San Diego Zoo'
                                            )).first()
            assert SDZ['Founded'] == datetime.date(1935, 9, 13)

    def _baseline_7_multiview(self):
        Zoo = self.metadata.tables['Zoo']
        Animal = self.metadata.tables['Animal']
        engine = self.metadata.bind

        def fulltable(select):
            """Iterate over the full result table."""

            return [list(row) for row in engine.execute(select).fetchall()]

        for x in range(ITERATIONS):
            fulltable(
                select(
                    [Zoo.c.ID] + list(Animal.c),
                    Zoo.c.Name == 'San Diego Zoo',
                    from_obj=[join(Zoo, Animal)]))
            Zoo.select(Zoo.c.Name == 'San Diego Zoo')
            fulltable(
                select(
                    [Zoo.c.ID, Animal.c.ID],
                    and_(
                        Zoo.c.Name == 'San Diego Zoo',
                        Animal.c.Species == 'Leopard'
                    ),
                    from_obj=[join(Zoo, Animal)])
            )

            # Now try the same query with INNER, LEFT, and RIGHT JOINs.

            fulltable(select([
                Zoo.c.Name, Animal.c.Species],
                from_obj=[join(Zoo, Animal)]))
            fulltable(select([
                Zoo.c.Name, Animal.c.Species],
                from_obj=[outerjoin(Zoo, Animal)]))
            fulltable(select([
                Zoo.c.Name, Animal.c.Species],
                from_obj=[outerjoin(Animal, Zoo)]))

    def _baseline_8_drop(self):
        self.metadata.drop_all()


from datasets import load_dataset
import matplotlib.pyplot as plt

rated_KoRAE = load_dataset("Cartinoe5930/KoRAE_rated", split="train")

score_result = {}
x_label = []
for i in range(0, 21):
    score_result[i * 0.5] = 0
    x_label.append(i * 0.5)

for data in rated_KoRAE:
    score_result[float(data["score"])] += 1

x = list(score_result.keys())
y = list(score_result.values())

plt.figure(figsize=(10, 6))
plt.bar(x, y, width=0.4, align='center', alpha=0.7)
plt.xlabel('Scores')
plt.ylabel('Frequency')
plt.title('Score Distribution Plot')
plt.xticks(x, rotation=45)
plt.grid(axis='y', linestyle='--', alpha=0.6)

plt.show()


# Basic idea:
# Physicsy sandy sand (pixel-perfect collisions, run simulation to keep up with real time)
# Character moving around
# Tap direction (WASD) to choose stuff to pick up, tap next to throw
# Hold direction to choose stuff to morph, press other keys to morph it

import pygame
from pygame.locals import *
import numpy as np
import time

pygame.init()

coltonum = lambda r, g, b: (((r << 8) + g) << 8) + b

w, h = 500, 250
screensize = np.int32((w, h))
screen = pygame.Surface(screensize)
window = pygame.display.set_mode(screensize * 2)

screengrid = pygame.surfarray.pixels3d(screen)
windowgrid = pygame.surfarray.pixels3d(window)

keys = set()

nx = 10
pos = np.int32([[*np.tile(np.arange(0, nx, 1, "int32"), 50), *np.tile(np.arange(300 - nx, 300, 1, "int32"), 50)],
                [*np.repeat(np.arange(0, 50, 1, "int32"), nx), *np.repeat(np.arange(0, 50, 1, "int32"), nx)]]).T
mass = np.ones(pos.shape[0], "int32")
vel = np.zeros(pos.shape, "int32")
vel[:pos.shape[0] // 2] = 20
mass[pos.shape[0] // 2:] = 10
force = np.zeros(pos.shape, "int32")

while True:
    screen.fill(0)
    screengrid[pos[:, 0], pos[:, 1], 0] = 255 - ((mass - 1) * 10)
    # try:
    #     screengrid[collpos[:, 0], collpos[:, 1], 1] = 255
    # except NameError:
    #     pass
    for i in range(2):
        for j in range(2):
            windowgrid[i::2, j::2] = screengrid
    pygame.display.flip()
    mousepos = np.int32(pygame.mouse.get_pos())
    for e in pygame.event.get():
        if e.type == QUIT:
            quit()
        elif e.type == KEYDOWN:
            keys.add(e.key)
            if e.key == K_ESCAPE:
                quit()
        elif e.type == KEYUP:
            keys.discard(e.key)
    # time.sleep(mousepos[0] / (10 * w))

    # In order to physics in discrete space, subdivide each tick according to lcm of integer speeds of objects
    #   Each object moves only in steps of 1 in x or y
    #   At the beginning of the appropriate subdivision of a tick, an object will move 1
    #   If, say, there are objects a and b moving at x speeds 2 and 3 (units per tick), the tick will be cut into 6:
    #       0: a  b      (this is the initial state)
    #       1: a  b
    #       2: a   b
    #       3:  a  b
    #       4:  a   b
    #       5:  a   b
    #       6:   a   b   (this is the state after the tick has passed)
    #   At each stage, collisions are checked and speeds altered, etc
    #       Which will... probably get complicated fast
    #           Each object will have to keep track of the subtick offset for its movement speed (last collision)
    #           The velocity composition (and hence lcm) may change with collisions
    #       Packed, resting objects will collide... a lot, which could be slow
    #           Maybe keep track of columns/rows locked against the sides?
    uniquev = np.unique(np.abs(vel))
    tickdiv = np.max(uniquev)
    # print(uniquev, tickdiv)
    # https://stackoverflow.com/a/42472824
    for v in uniquev[(tickdiv % uniquev) != 0]:
        # print(v, tickdiv, np.gcd(tickdiv, v))
        tickdiv = int(tickdiv * v / np.gcd(tickdiv, v))
    # print("F", tickdiv)
    i = 1
    while i < tickdiv:
        movers = ((tickdiv / i) % np.abs(vel)) == 0
        # print(i, tickdiv, vel[movers])
        pos[movers] += np.sign(vel[movers])
        vel[((pos == 0) & (vel < 0)) | ((pos == (w, h)) & (vel > 0))] *= -1
        pos = np.minimum(np.maximum(0, pos), (w - 1, h - 1))
        # https://stackoverflow.com/q/11528078
        a = np.lexsort([*pos.T])
        coll = np.all(pos[a[:-1]] == pos[a[1:]], axis=1)
        # print(sum(coll))
        if np.any(coll):
            collpos = pos[a[:-1][coll]]
            # print(pos[a[0]], pos[a[-1]])
            # Assume that there are only 2 parties to each collision, and that all masses are equal...
            oppcoll = (vel[a[:-1][coll]] * vel[a[1:][coll]]) < 0
            # print(sum(oppcoll))
            newvel = vel.copy()
            # print(vel.shape, a.shape, oppcoll.shape)
            newvel[a[:-1][coll][oppcoll[:, 0]], 0] = vel[a[1:][coll][oppcoll[:, 0]], 0]
            newvel[a[:-1][coll][oppcoll[:, 1]], 1] = vel[a[1:][coll][oppcoll[:, 1]], 1]
            newvel[a[1:][coll][oppcoll[:, 0]], 0] = vel[a[:-1][coll][oppcoll[:, 0]], 0]
            newvel[a[1:][coll][oppcoll[:, 1]], 1] = vel[a[:-1][coll][oppcoll[:, 1]], 1]

            newvel[a[:-1][coll][~oppcoll[:, 0]], 0] = vel[a[1:][coll][~oppcoll[:, 0]], 0]
            newvel[a[:-1][coll][~oppcoll[:, 1]], 1] = vel[a[1:][coll][~oppcoll[:, 1]], 1]
            newvel[a[1:][coll][~oppcoll[:, 0]], 0] = vel[a[:-1][coll][~oppcoll[:, 0]], 0]
            newvel[a[1:][coll][~oppcoll[:, 1]], 1] = vel[a[:-1][coll][~oppcoll[:, 1]], 1]
            # print(time.time(), np.all(newvel == vel), newvel is vel)
            vel[:, :] = newvel[:, :]
            if not np.all((tickdiv % vel) == 0):
                p = i / tickdiv
                uniquev = np.unique(np.abs(vel))
                for v in uniquev[(tickdiv % uniquev) != 0]:
                    tickdiv = int(tickdiv * v / np.gcd(tickdiv, v))
                # print(tickdiv)
                i = np.floor(p * tickdiv)
        i += 1
    # vel -= np.int32(np.ceil(vel / 100))


from src.depth_transform import *
from src.audio_transform import *
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
from src.dataset_utils import _load_audio_file

MAX_DEPTH = 10000
MIN_DEPTH = 0.0

def test_depth_arr(depth):
    params = {"max_depth": 10000}
    depth = transform_depth(depth, params)
    return depth

if __name__ == "__main__":
    #Image
    IMAGE_EXAMPLE = Image.open("camera_47.jpeg")

    #Depth
    DEPTH_EXAMPLE = np.load("depth_47.npy")
    DEPTH_EXAMPLE = test_depth_arr(DEPTH_EXAMPLE)
    DEPTH_IMG = Image.fromarray(DEPTH_EXAMPLE)

    #Audio
    params = {"feature_name": "spectrogram", "n_fft": 512, "power": 1, "win_length": 128, "hop_length": 64, "to_db": False}
    AUDIO_EXAMPLE, sr = _load_audio_file("audio_47.wav", sr=44100, max_depth=MAX_DEPTH)
    AUDIO_SPEC = transform_audio(AUDIO_EXAMPLE, feature_extraction_params=params)

    fig, axs = plt.subplots(1, 3)

    im1 = axs[0].imshow(IMAGE_EXAMPLE)
    axs[0].axis('off')
    axs[0].set_title("Original Image")

    im2 = axs[1].imshow(DEPTH_IMG, vmin=0, vmax=1)
    axs[1].axis('off')
    axs[1].set_title("Depth")
    cbar2 = fig.colorbar(im2, ax=axs[1])

    im3 = axs[2].imshow(AUDIO_SPEC, aspect="auto", origin="lower")
    axs[2].set_title("Spectrogram")
    cbar3 = fig.colorbar(im3, ax=axs[2])

    plt.subplots_adjust(wspace=0.3)

    plt.show()


import os
from pprint import pprint

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

from app_ml.functionalities.constants import SPARK_WINDOW, MONGO_URL, WINDOW, SAVED_MODEL_PATH, POSTGRES_URL
from app_ml.functionalities.preprocessing import create_test_set
from app_ml.models.RNN import RNN
from sqlalchemy import create_engine, insert, MetaData, Table
from collections import Counter
import pymongo
import tensorflow as tf
import numpy as np
import datetime

DATES = [datetime.datetime(2021, 2, 17, 9, 13, 27), datetime.datetime(2021, 2, 18, 21, 33, 2)]

FILES = [
    'opel_corsa_01',
    'opel_corsa_02'
    # 'peugeot_207_01'
    # 'peugeot_207_02'
]

def read_driver_route_id_from_mongo():
    cars_db = pymongo.MongoClient(MONGO_URL)['cars']
    routes_collection = cars_db['routes']
    cursor = routes_collection.aggregate(
        [
            {'$group': {'_id': {'DriverId': '$DriverId', 'RouteId': '$RouteId'}}}
        ]
    )
    driver_route_ids = [(d['_id']['DriverId'], d['_id']['RouteId']) for d in cursor]
    return driver_route_ids

def predict_for_file(model, file):
    x_test, y_test = create_test_set(WINDOW, file)
    x_test, _ = model.preprocess_test(x_test, y_test)
    predictions = model.model.predict(x_test)
    results = np.argmax(predictions, axis=1)
    return results

def make_scores(results):
    results_count = Counter(results)
    aggressive_count = 0 if results_count.get(0) is None else results_count.get(0)
    normal_count = 0 if results_count.get(1) is None else results_count.get(1)
    total_count = aggressive_count + normal_count
    return aggressive_count / total_count, normal_count / total_count

def make_postgres_data_list(route_id, results):
    data = []
    start = 10
    for i in range(0, len(results)):
        end = start + SPARK_WINDOW - 1
        aggressive_score, normal_score = make_scores(results[i])
        d = {
                'route_id': route_id, 'start_time': start, 'end_time': end,
                'aggressive_score': aggressive_score, 'normal_score': normal_score
            }
        data.append(d)
        start = end + 1
    return data

def insert_to_postgres_db(route_id, driver_id, date, model, engine, file):
    results = predict_for_file(model, file)
    spark_windowed_results = [results[i:SPARK_WINDOW+i] for i in range(0, len(results) - SPARK_WINDOW, SPARK_WINDOW)]
    data = make_postgres_data_list(route_id, spark_windowed_results)
    meta = MetaData(engine)
    routes = Table('routes', meta, autoload=True)
    scores = Table('scores', meta, autoload=True)
    routes_stmt = insert(routes).values({'route_id': route_id, 'user_id': driver_id, 'date': date, 'is_active': 0})
    scores_stmt = insert(scores).values(data)
    conn = engine.connect()
    conn.execute(routes_stmt)
    conn.execute(scores_stmt)
    conn.execute('COMMIT')
    conn.close()

def main():
    driver_route_ids = read_driver_route_id_from_mongo()
    model = RNN((20, 15), 2)
    model.model = tf.keras.models.load_model(SAVED_MODEL_PATH, compile=False)
    engine = create_engine(POSTGRES_URL)
    for i in range(0, len(FILES)):
        insert_to_postgres_db(driver_route_ids[i][1], driver_route_ids[i][0], DATES[i], model, engine, FILES[i])
    engine.dispose()
    print('Data inserted successfully to PostgresDB')

if __name__ == '__main__':
    main()


from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import GridSearchCV
import catboost as cb
import xgboost as xgb
import pandas as pd

def gradientBoost(X,y):
    gboost=GradientBoostingRegressor(random_state=11)
    params={'max_depth':[1,2,3],'n_estimators':[100,300,500,1000]}
    search=GridSearchCV(estimator=gboost,param_grid=params,scoring='r2',cv=3,return_train_score=True,verbose=2)
    search.fit(X,y)
    print('Best parameters for Gradient Boosting are:', search.best_params_)
    return search.best_estimator_

def catboost(X,y):
    cboost=cb.CatBoostRegressor(silent=True)
    params={'learning_rate':[0.1,0.5,0.01,0.05],'iterations':[100,300,500,1000],'depth':[1,2]}
    search=GridSearchCV(estimator=cboost,param_grid=params,scoring='r2',cv=3,return_train_score=True,verbose=2)
    search.fit(X,y)
    print('Best parameters for CatBoost are:', search.best_params_)
    return search.best_estimator_

def xgboost(X,y):
    xgboost=xgb.XGBRegressor(objective ='reg:linear',enable_cateorical=True)
    params={'learning_rate':[0.1,0.5,0.01,1],'n_estimators':[10,30,100,500],'colsample_bytree':[0.3,0.5],
                'max_depth':[1,2,3,4,5], 'alpha': [0.1,1,0.01,10]}
    search=GridSearchCV(estimator=xgboost,param_grid=params,scoring='r2',cv=3,return_train_score=True,verbose=1)
    search.fit(X,y)
    print('Best parameters for XgBoost are:', search.best_params_)
    return search.best_estimator_