"""Append result to shapefile
"""
import logging
import shapefile

def write_result_shapefile(lad_geometry_shp, out_shape, field_names, csv_results):
    """
    Join result attributes to LAD geography with
    pyhape library

    Arguments
    ---------
    lad_geometry_shp : str
        Path to LAD shapefile
    out_shape : str
        Path to new shapefile
    field_names : list
        list with new attribute field name
    csv_results : list
        list with result dicts

    Info
    -----
    pip install pyshp
    https://github.com/GeospatialPython/pyshp#reading-shapefiles-from-file-like-objects

    http://www.qgistutorials.com/en/docs/performing_table_joins_pyqgis.html
    """
    # Read in our existing shapefile
    lad_geometry_shp_name = lad_geometry_shp[:-3]
    myshp = open(lad_geometry_shp_name + "shp", "rb")
    mydbf = open(lad_geometry_shp_name + "dbf", "rb")

    record = shapefile.Reader(shp=myshp, dbf=mydbf)

    # Create a new shapefile in memory
    writer = shapefile.Writer()

    # Copy over the existing fields
    writer.fields = list(record.fields)

    # --------------
    # Add new fields
    # --------------
    for field_name in field_names:
        writer.field(field_name, "F", decimal=10) #Float

    # Get position of field 'name'
    position = 0
    for field_name in record.fields[1:]:
        if field_name[0] == 'name': #corresponds to LAD Geocode
            position_field_name = position
            break
        else:
            position += 1

    # --------------------------
    # Join fields programatically
    # --------------------------
    missing_recors = set()

    # Loop through each record, add a column and get results
    for rec in record.records():

        # Get geocode for row
        geo_code = rec[position_field_name]

        # Iterate result entries in list
        for result_per_field in csv_results:

            # Iterate result entries and add
            try:
                result_csv = result_per_field[geo_code]
            except KeyError:
                # No results
                result_csv = 0
                missing_recors.add(geo_code)

            # Add specific fuel result
            rec.append(result_csv)

        # Add the modified record to the new shapefile
        writer.records.append(rec)

    if missing_recors != []:
        logging.warning(
            "No result value for regions '%s' in joining shapefile",
            missing_recors)
    else:
        pass

    # Copy over the geometry without any changes
    writer._shapes.extend(record.shapes())

    # Save as a new shapefile (or write over the old one)
    writer.save(out_shape)
    logging.info("... finished writing shp")
    return


#   Coyright 2017-2019 Nativepython Authors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from typed_python import (
    Bool,
    Int8, Int16, Int32, Int64,
    UInt8, UInt16, UInt32, UInt64,
    Float32, Float64,
    NoneType, TupleOf, ListOf, OneOf, Tuple, NamedTuple, Dict,
    ConstDict, Alternative, serialize, deserialize, Class, Member,
    TypeFilter, Function, Forward
)

from typed_python.test_util import currentMemUsageMb
import typed_python._types as _types
import psutil
import unittest
import time
import numpy
import os

def typeFor(t):
    assert not isinstance(t, list), t
    return type(t)

def typeForSeveral(t):
    ts = set(typeFor(a) for a in t)
    if len(ts) == 1:
        return list(ts)[0]
    return OneOf(*ts)

def makeTupleOf(*args):
    if not args:
        return TupleOf(int)()
    return TupleOf(typeForSeveral(args))(args)

def makeNamedTuple(**kwargs):
    if not kwargs:
        return NamedTuple()()
    return NamedTuple(**{k: typeFor(v) for k, v in kwargs.items()})(**kwargs)

def makeTuple(*args):
    if not args:
        return Tuple()()
    return Tuple(*[typeFor(v) for v in args])(args)

def makeDict(d):
    if not d:
        return ConstDict(int, int)()

    return ConstDict(typeForSeveral(d.keys()), typeForSeveral(d.values()))(d)

def makeAlternative(severalDicts):
    types = list(
        set(
            tuple(
                (k, typeFor(v)) for k, v in ntDict.items()
            )
            for ntDict in severalDicts
        )
    )

    alt = Alternative("Alt", **{
        "a_%s" % i: dict(types[i]) for i in range(len(types))
    })

    res = []
    for thing in severalDicts:
        did = False
        for i in range(len(types)):
            try:
                res.append(getattr(alt, "a_%s" % i)(**thing))
                did = True
            except Exception:
                pass

            if did:
                break
    assert len(res) == len(severalDicts)

    return res

def choice(x):
    # numpy.random.choice([1,(1,2)]) blows up because it looks 'multidimensional'
    # so we have to pick from a list of indices
    if not isinstance(x, list):
        x = list(x)
    return x[numpy.random.choice(list(range(len(x))))]

class RandomValueProducer:
    def __init__(self):
        self.levels = {0: [b'1', b'', '2', '', 0, 1, 0.0, 1.0, None, False, True, "a ", "a string", "b string", "b str"]}

    def addEvenly(self, levels, count):
        for level in range(1, levels+1):
            self.addValues(level, count)

    def all(self):
        res = []
        for valueList in self.levels.values():
            res.extend(valueList)
        return res

    def addValues(self, level, count, sublevels=None):
        assert level > 0

        if sublevels is None:
            sublevels = list(range(level))
        sublevels = [x for x in sublevels if x in self.levels]

        assert sublevels

        def picker():
            whichLevel = choice(sublevels)
            try:
                return choice(self.levels[whichLevel])
            except Exception:
                print(self.levels[whichLevel])
                raise

        for _ in range(count):
            val = self.randomValue(picker)
            if not isinstance(val, list):
                val = [val]
            self.levels.setdefault(level, []).extend(val)

    def randomValue(self, picker):
        def randomTuple():
            return makeTuple(*[picker() for i in range(choice([0, 1, 2, 3, 4]))])

        def randomNamedTupleDict():
            return {"x_%s" % i: picker() for i in range(choice([0, 1, 2, 3, 4]))}

        def randomNamedTuple():
            return makeNamedTuple(**randomNamedTupleDict())

        def randomDict():
            return makeDict({picker(): picker() for i in range(choice([0, 1, 2, 3, 4]))})

        def randomTupleOf():
            return makeTupleOf(*[picker() for i in range(choice([0, 1, 2, 3, 4]))])

        def randomAlternative():
            return makeAlternative([randomNamedTupleDict() for i in range(choice([1, 2, 3, 4]))])

        return choice([randomTuple, randomNamedTuple, randomDict, randomTupleOf, randomAlternative, picker])()

    def pickRandomly(self):
        return choice(self.levels[choice(list(self.levels))])

class NativeTypesTests(unittest.TestCase):

    def check_expected_performance(self, elapsed, expected=1.0):
        if os.environ.get('TRAVIS_CI', None) is not None:
            expected = 2 * expected

        self.assertTrue(
            elapsed < expected,
            "Slow Performance: expected to take {expected} sec, but took {elapsed}"
            .format(expected=expected, elapsed=elapsed)
        )

    def test_objects_are_singletons(self):
        self.assertTrue(Int8 is Int8)
        self.assertTrue(NoneType is NoneType)

    def test_object_binary_compatibility(self):
        ibc = _types.isBinaryCompatible

        self.assertTrue(ibc(NoneType, NoneType))
        self.assertTrue(ibc(Int8, Int8))

        NT = NamedTuple(a=int, b=int)

        class X(NamedTuple(a=int, b=int)):
            pass

        class Y(NamedTuple(a=int, b=int)):
            pass

        self.assertTrue(ibc(X, X))
        self.assertTrue(ibc(X, Y))
        self.assertTrue(ibc(X, NT))
        self.assertTrue(ibc(Y, NT))
        self.assertTrue(ibc(NT, Y))

        self.assertFalse(ibc(OneOf(int, float), OneOf(float, int)))
        self.assertTrue(ibc(OneOf(int, X), OneOf(int, Y)))

    def test_binary_compatibility_incompatible_alternatives(self):
        ibc = _types.isBinaryCompatible

        A1 = Alternative("A1", X={'a': int}, Y={'b': float})
        A2 = Alternative("A2", X={'a': int}, Y={'b': str})

        self.assertTrue(ibc(A1, A1.X))
        self.assertTrue(ibc(A1, A1.Y))
        self.assertTrue(ibc(A1.Y, A1.Y))
        self.assertTrue(ibc(A1.Y, A1))
        self.assertTrue(ibc(A1.X, A1))
        self.assertFalse(ibc(A1.X, A1.Y))

        self.assertFalse(ibc(A1, A2))
        self.assertFalse(ibc(A1.X, A2.X))
        self.assertFalse(ibc(A1.Y, A2.Y))

    def test_binary_compatibility_compatible_alternatives(self):
        ibc = _types.isBinaryCompatible

        A1 = Alternative("A1", X={'a': int}, Y={'b': float})
        A2 = Alternative("A2", X={'a': int}, Y={'b': float})

        self.assertTrue(ibc(A1.X, A2.X))
        self.assertTrue(ibc(A1.Y, A2.Y))

        self.assertFalse(ibc(A1.X, A2.Y))
        self.assertFalse(ibc(A1.Y, A2.X))

    def test_callable_alternatives(self):
        def myCall(self):
            if self.matches.One:
                return 1
            elif self.matches.Two:
                return 2
            else:
                raise TypeError("Unexpected alternative kind")

        alt = Alternative("alts", One={}, Two={}, __call__=myCall)

        one = alt.One()
        self.assertEqual(one(), 1)

        two = alt.Two()
        self.assertEqual(two(), 2)

        alt = Alternative("alts", One={}, Two={}, myCall=myCall)
        with self.assertRaises(TypeError):
            one = alt.One()
            one()

        with self.assertRaises(TypeError):
            two = alt.Two()
            two()

    def test_callable_class(self):
        class CallableClass(Class):
            x = Member(int)

            def __call__(self, x):
                return self.x + x

            def __call__(self):  # noqa: F811
                return -1

        class RegularClass(Class):
            x = Member(int)

            def call(self, x):
                return self.x + x

        obj = CallableClass(x=42)
        self.assertEqual(obj(0), 42)
        self.assertEqual(obj(1), 43)
        self.assertEqual(obj(), -1 )

        exceptionMsg = "Cannot find a valid overload of '__call__' with arguments of type"
        with self.assertRaisesRegex(TypeError, exceptionMsg):
            obj(1, 2, 3)

        obj = RegularClass(x=42)
        self.assertEqual(obj.call(5), 47)
        with self.assertRaises(TypeError):
            obj()

    def test_object_bytecounts(self):
        self.assertEqual(_types.bytecount(NoneType), 0)
        self.assertEqual(_types.bytecount(Int8), 1)
        self.assertEqual(_types.bytecount(Int64), 8)

    def test_type_stringification(self):
        for t in ['Int8', 'NoneType']:
            self.assertEqual(str(getattr(_types, t)()), "<class '%s'>" % t)

    def test_tuple_of(self):
        tupleOfInt = TupleOf(int)
        i = tupleOfInt(())
        i = tupleOfInt((1, 2, 3))

        self.assertEqual(len(i), 3)
        self.assertEqual(tuple(i), (1, 2, 3))

        for x in range(10):
            self.assertEqual(
                tuple(tupleOfInt(tuple(range(x)))),
                tuple(range(x))
            )

        with self.assertRaisesRegex(AttributeError, "do not accept attributes"):
            tupleOfInt((1, 2, 3)).x = 2

    def test_one_of_alternative(self):
        X = Alternative("X", V={'a': int})
        Ox = OneOf(None, X)

        self.assertEqual(Ox(X.V(a=10)), X.V(a=10))

    def test_one_of_py_subclass(self):
        class X(NamedTuple(x=int)):
            def f(self):
                return self.x

        Ox = OneOf(None, X)

        self.assertEqual(NamedTuple(x=int)(x=10).x, 10)
        self.assertEqual(X(x=10).f(), 10)
        self.assertEqual(Ox(X(x=10)).f(), 10)

    def test_one_of_distinguishes_py_subclasses(self):
        class X(NamedTuple(x=int)):
            def f(self):
                return self.x

        class X2(NamedTuple(x=int)):
            def f(self):
                return self.x + 2

        XorX2 = OneOf(X, X2)

        self.assertTrue(isinstance(XorX2(X()), X))
        self.assertTrue(isinstance(XorX2(X2()), X2))

    def test_tuple_of_tuple_of(self):
        tupleOfInt = TupleOf(int)
        tupleOfTupleOfInt = TupleOf(tupleOfInt)

        pyVersion = (1, 2, 3), (1, 2, 3, 4)
        nativeVersion = tupleOfTupleOfInt(pyVersion)

        self.assertEqual(len(nativeVersion), 2)
        self.assertEqual(len(nativeVersion[0]), 3)
        self.assertEqual(tuple(tuple(x) for x in nativeVersion), pyVersion)

        bigTup = tupleOfInt(list(range(1000)))

        t0 = time.time()
        t = (bigTup, bigTup, bigTup, bigTup, bigTup)
        for i in range(1000000):
            tupleOfTupleOfInt(t)

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_default_initializer_oneof(self):
        x = OneOf(None, int)
        self.assertTrue(x() is None, repr(x()))

    def test_tuple_of_various_things(self):
        for thing, typeOfThing in [("hi", str), (b"somebytes", bytes),
                                   (1.0, float), (2, int),
                                   (None, type(None))
                                   ]:
            tupleType = TupleOf(typeOfThing)
            t = tupleType((thing,))
            self.assertTrue(type(t[0]) is typeOfThing)
            self.assertEqual(t[0], thing)

    def test_tuple_assign_fails(self):
        with self.assertRaisesRegex(TypeError, "does not support item assignment"):
            (1, 2, 3)[10] = 20
        with self.assertRaisesRegex(TypeError, "does not support item assignment"):
            TupleOf(int)((1, 2, 3))[10] = 20

    def test_list_of(self):
        L = ListOf(int)
        self.assertEqual(L.__qualname__, "ListOf(Int64)")

        l1 = L([1, 2, 3, 4])

        self.assertEqual(l1[0], 1)
        self.assertEqual(l1[-1], 4)

        l1[0] = 10
        self.assertEqual(l1[0], 10)

        l1[-1] = 11
        self.assertEqual(l1[3], 11)

        with self.assertRaisesRegex(IndexError, "index out of range"):
            l1[100] = 20

        l2 = L((10, 2, 3, 11))

        self.assertEqual(l1, l2)
        self.assertNotEqual(l1, (10, 2, 3, 11))
        self.assertEqual(l1, [10, 2, 3, 11])

        self.assertEqual(str(l1), str([10, 2, 3, 11]))

        l3 = l1 + l2
        self.assertEqual(l3, [10, 2, 3, 11, 10, 2, 3, 11])

        l3.append(23)
        self.assertEqual(l3, [10, 2, 3, 11, 10, 2, 3, 11, 23])

    def test_list_resize(self):
        l1 = ListOf(TupleOf(int))()

        l1.resize(10)
        self.assertEqual(l1.reserved(), 10)
        self.assertEqual(len(l1), 10)

        emptyTup = TupleOf(int)()
        aTup = TupleOf(int)((1, 2, 3))

        self.assertEqual(list(l1), [emptyTup] * 10)
        l1.resize(20, aTup)
        self.assertEqual(list(l1), [emptyTup] * 10 + [aTup] * 10)

        self.assertEqual(_types.refcount(aTup), 11)

        self.assertEqual(l1.pop(15), aTup)
        self.assertEqual(l1.pop(5), emptyTup)

        self.assertEqual(_types.refcount(aTup), 10)

        l1.resize(15)

        with self.assertRaises(IndexError):
            l1.pop(100)

        self.assertEqual(_types.refcount(aTup), 7)  # 6 in the list because we popped at '5'

        l1.pop()

        self.assertEqual(_types.refcount(aTup), 6)

        # this pops one of the empty tuples
        l1.pop(-10)

        self.assertEqual(_types.refcount(aTup), 6)

        l1.clear()
        self.assertEqual(len(l1), 0)

    def test_one_of(self):
        o = OneOf(None, str)

        self.assertEqual(o("hi"), "hi")
        self.assertTrue(o(None) is None)

        o = OneOf(None, "hi", 1.5, 1, True, b"hi2")

        self.assertTrue(o(None) is None)
        self.assertTrue(o("hi") == "hi")
        self.assertTrue(o(b"hi2") == b"hi2")
        self.assertTrue(o(1.5) == 1.5)
        self.assertTrue(o(1) == 1)
        self.assertIs(o(True), True)

        with self.assertRaises(TypeError):
            o("hi2")
        with self.assertRaises(TypeError):
            o(b"hi")
        with self.assertRaises(TypeError):
            o(3)
        with self.assertRaises(TypeError):
            o(False)

    def test_ordering(self):
        o = OneOf(None, "hi", 1.5, 1, True, b"hi2")

        self.assertIs(o(True), True)

    def test_one_of_flattening(self):
        self.assertEqual(OneOf(OneOf(None, 1.0), OneOf(2.0, 3.0)), OneOf(None, 1.0, 2.0, 3.0))

    def test_one_of_order_matters(self):
        self.assertNotEqual(OneOf(1.0, 2.0), OneOf(2.0, 1.0))

    def test_type_filter(self):
        EvenInt = TypeFilter(int, lambda i: i % 2 == 0)

        self.assertTrue(isinstance(2, EvenInt))
        self.assertFalse(isinstance(1, EvenInt))
        self.assertFalse(isinstance(2.0, EvenInt))

        EvenIntegers = TupleOf(EvenInt)

        e = EvenIntegers(())
        e2 = e + (2, 4, 0)

        with self.assertRaises(TypeError):
            EvenIntegers((1,))

        with self.assertRaises(TypeError):
            e2 + (1,)

    def test_tuple_of_one_of_fixed_size(self):
        t = TupleOf(OneOf(0, 1, 2, 3, 4))

        ints = tuple([x % 5 for x in range(1000000)])

        typedInts = t(ints)

        self.assertEqual(len(serialize(t, typedInts)), len(ints) * 2 + 6)  # 3 bytes for extra flags
        self.assertEqual(tuple(typedInts), ints)

    def test_tuple_of_one_of_multi(self):
        t = TupleOf(OneOf(int, bool))

        someThings = tuple([100 + x % 5 if x % 17 != 0 else bool(x%19) for x in range(1000000)])

        typedThings = t(someThings)

        self.assertEqual(
            len(serialize(t, typedThings)),
            sum(3 if isinstance(t, bool) else 4 for t in someThings) +
            2 +  # two bytes for begin / end flags
            2 +  # two bytes for the id
            2    # two bytes for the size
        )

        self.assertEqual(tuple(typedThings), someThings)

    def test_compound_oneof(self):
        producer = RandomValueProducer()
        producer.addEvenly(1000, 2)

        for _ in range(1000):
            vals = (producer.pickRandomly(), producer.pickRandomly(), producer.pickRandomly())

            a = OneOf(vals[0], vals[1], type(vals[2]))

            for v in vals:
                self.assertEqual(a(v), v, (a(v), v))

            tup = TupleOf(a)
            tupInst = tup(vals)

            for i in range(len(vals)):
                self.assertEqual(tupInst[i], vals[i], vals)

    def test_one_of_conversion_failure(self):
        o = OneOf(None, str)

        with self.assertRaises(TypeError):
            o(b"bytes")

    def test_one_of_in_tuple(self):
        t = Tuple(OneOf(None, str), str)

        self.assertEqual(t(("hi", "hi2"))[0], "hi")
        self.assertEqual(t(("hi", "hi2"))[1], "hi2")
        self.assertEqual(t((None, "hi2"))[1], "hi2")
        self.assertEqual(t((None, "hi2"))[0], None)
        with self.assertRaises(TypeError):
            t((None, None))
        with self.assertRaises(IndexError):
            t((None, "hi2"))[2]

    def test_one_of_composite(self):
        t = OneOf(TupleOf(str), TupleOf(float))

        self.assertIsInstance(t((1.0, 2.0)), TupleOf(float))
        self.assertIsInstance(t(("1.0", "2.0")), TupleOf(str))

        with self.assertRaises(TypeError):
            t((1.0, "2.0"))

    def test_named_tuple(self):
        t = NamedTuple(a=int, b=int)

        with self.assertRaisesRegex(AttributeError, "object has no attribute"):
            t().asdf

        with self.assertRaisesRegex(AttributeError, "immutable"):
            t().a = 1

        self.assertEqual(t()[0], 0)
        self.assertEqual(t().a, 0)
        self.assertEqual(t()[1], 0)

        self.assertEqual(t(a=1, b=2).a, 1)
        self.assertEqual(t(a=1, b=2).b, 2)

    def test_named_tuple_construction(self):
        t = NamedTuple(a=int, b=int)

        self.assertEqual(t(a=10).a, 10)
        self.assertEqual(t(a=10).b, 0)
        self.assertEqual(t(a=10, b=2).a, 10)
        self.assertEqual(t(a=10, b=2).b, 2)
        self.assertEqual(t({'a': 10, 'b': 2}).a, 10)
        self.assertEqual(t({'a': 10, 'b': 2}).b, 2)

        self.assertEqual(t({'b': 2}).a, 0)
        self.assertEqual(t({'b': 2}).b, 2)

        with self.assertRaises(TypeError):
            t({'c': 10})
        with self.assertRaises(TypeError):
            t(c=10)

    def test_named_tuple_str(self):
        t = NamedTuple(a=str, b=str)

        self.assertEqual(t(a='1', b='2').a, '1')
        self.assertEqual(t(a='1', b='2').b, '2')

        self.assertEqual(t(b='2').a, '')
        self.assertEqual(t(b='2').b, '2')
        self.assertEqual(t().a, '')
        self.assertEqual(t().b, '')

    def test_tuple_of_string_perf(self):
        t = NamedTuple(a=str, b=str)

        t0 = time.time()
        for i in range(1000000):
            t(a="a", b="b").a

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_comparisons_in_one_of(self):
        t = OneOf(None, float)

        def map(x):
            if x is None:
                return -1000000.0
            else:
                return x

        lt = lambda a, b: map(a) < map(b)
        le = lambda a, b: map(a) <= map(b)
        eq = lambda a, b: map(a) == map(b)
        ne = lambda a, b: map(a) != map(b)
        gt = lambda a, b: map(a) > map(b)
        ge = lambda a, b: map(a) >= map(b)

        funcs = [lt, le, eq, ne, gt, ge]
        ts = [None, 1.0, 2.0, 3.0]

        for f in funcs:
            for t1 in ts:
                for t2 in ts:
                    self.assertTrue(f(t1, t2) is f(t(t1), t(t2)))

    def test_comparisons_equivalence(self):
        t = TupleOf(OneOf(None, str, bytes, float, int, bool, TupleOf(int)),)

        def lt(a, b):
            return a < b

        def le(a, b):
            return a <= b

        def eq(a, b):
            return a == b

        def ne(a, b):
            return a != b

        def gt(a, b):
            return a > b

        def ge(a, b):
            return a >= b

        funcs = [lt, le, eq, ne, gt, ge]

        tgroups = [
            [1.0, 2.0, 3.0],
            [1, 2, 3],
            [True, False],
            ["a", "b", "ab", "bb", "ba", "aaaaaaa", "", "asdf"],
            ["1", "2", "3", "12", "13", "23", "24", "123123", "0", ""],
            [b"a", b"b", b"ab", b"bb", b"ba", b"aaaaaaa", b"", b"asdf"],
            [(1, 2), (1, 2, 3), (), (1, 1), (1,)]
        ]

        for ts in tgroups:
            for f in funcs:
                for t1 in ts:
                    for t2 in ts:
                        self.assertTrue(
                            f(t1, t2) is f(t((t1,)), t((t2,))),
                            (f, t1, t2, f(t1, t2), f(t((t1,)), t((t2,))))
                        )

    def test_const_dict(self):
        t = ConstDict(str, str)

        self.assertEqual(len(t()), 0)
        self.assertEqual(len(t({})), 0)
        self.assertEqual(len(t({'a': 'b'})), 1)
        self.assertEqual(t({'a': 'b'})['a'], 'b')
        self.assertEqual(t({'a': 'b', 'b': 'c'})['b'], 'c')

        self.assertTrue("a" in deserialize(t, serialize(t, t({'a': 'b'}))))

        self.assertTrue("a" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c'}))))
        self.assertTrue("a" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c', 'c': 'd'}))))
        self.assertTrue("a" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c', 'c': 'd', 'd': 'e'}))))
        self.assertTrue("c" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c', 'c': 'd', 'def': 'e'}))))
        self.assertTrue("def" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c', 'c': 'd', 'def': 'e'}))))

    def test_const_dict_get(self):
        a = ConstDict(str, str)({'a': 'b', 'c': 'd'})

        self.assertEqual(a.get('a'), 'b')
        self.assertEqual(a.get('asdf'), None)
        self.assertEqual(a.get('asdf', 20), 20)

    def test_const_dict_items_keys_and_values(self):
        a = ConstDict(str, str)({'a': 'b', 'c': 'd'})

        self.assertEqual(sorted(a.items()), [('a', 'b'), ('c', 'd')])
        self.assertEqual(sorted(a.keys()), ['a', 'c'])
        self.assertEqual(sorted(a.values()), ['b', 'd'])

    def test_empty_string(self):
        a = ConstDict(str, str)({'a': ''})

        print(a['a'])

    def test_dict_to_oneof(self):
        t = ConstDict(str, OneOf("A", "B", "ABCDEF"))
        a = t({'a': 'A', 'b': 'ABCDEF'})

        self.assertEqual(a['a'], "A")
        self.assertEqual(a['b'], "ABCDEF")

        self.assertEqual(a, deserialize(t, serialize(t, a)))

    def test_deserialize_primitive(self):
        x = deserialize(str, serialize(str, "a"))
        self.assertTrue(isinstance(x, str))

    def test_dict_containment(self):
        for _ in range(100):
            producer = RandomValueProducer()
            producer.addEvenly(20, 2)

            values = producer.all()

            for v in values:
                if str(type(v))[:17] == "<class 'ConstDict":
                    v = deserialize(type(v), serialize(type(v), v))
                    for k in v:
                        self.assertTrue(k in v)

    def test_named_tuple_from_dict(self):
        N = NamedTuple(x=int, y=str, z=OneOf(None, "hihi"))
        self.assertEqual(N().x, 0)
        self.assertEqual(N().y, "")
        self.assertEqual(N().z, None)

        self.assertEqual(N({}).x, 0)
        self.assertEqual(N({}).y, "")
        self.assertEqual(N({}).z, None)

        self.assertEqual(N({'x': 20}).x, 20)
        self.assertEqual(N({'x': 20, 'y': "30"}).y, "30")
        self.assertEqual(N({'y': "30", 'x': 20}).y, "30")
        self.assertEqual(N({'z': "hihi"}).z, "hihi")

        with self.assertRaises(Exception):
            N({'r': 'hi'})
            N({'y': 'hi', 'z': "not hihi"})
            N({'a': 0, 'b': 0, 'c': 0, 'd': 0})

    def test_const_dict_mixed(self):
        t = ConstDict(str, int)
        self.assertTrue(t({"a": 10})["a"] == 10)

        t = ConstDict(int, str)
        self.assertTrue(t({10: "a"})[10] == "a")

    def test_const_dict_comparison(self):
        t = ConstDict(str, str)

        self.assertEqual(t({'a': 'b'}), t({'a': 'b'}))
        self.assertLess(t({}), t({'a': 'b'}))

    def test_const_dict_lookup(self):
        for type_to_use, vals in [(int, list(range(20))),
                                  (bytes, [b'1', b'2', b'3', b'4', b'5'])]:
            t = ConstDict(type_to_use, type_to_use)

            for _ in range(10):
                ks = list(vals)
                vs = list(vals)

                numpy.random.shuffle(ks)
                numpy.random.shuffle(vs)

                py_d = {}
                for i in range(len(ks)):
                    py_d[ks[i]] = vs[i]

                typed_d = t(py_d)

                for k in py_d:
                    self.assertEqual(py_d[k], typed_d[k])

                last_k = None
                for k in typed_d:
                    assert last_k is None or k > last_k, (k, last_k)
                    last_k = k

    def test_const_dict_lookup_time(self):
        int_dict = ConstDict(int, int)

        d = int_dict({k: k for k in range(1000000)})

        for k in range(1000000):
            self.assertTrue(k in d)
            self.assertTrue(d[k] == k)

    def test_const_dict_of_dict(self):
        int_dict = ConstDict(int, int)
        int_dict_2 = ConstDict(int_dict, int_dict)

        d = int_dict({1: 2})
        d2 = int_dict({1: 2, 3: 4})

        big = int_dict_2({d: d2})

        self.assertTrue(d in big)
        self.assertTrue(d2 not in big)
        self.assertTrue(big[d] == d2)

    def test_dict_hash_perf(self):
        str_dict = ConstDict(str, str)

        s = str_dict({'a' * 1000000: 'b' * 1000000})

        t0 = time.time()
        for k in range(1000000):
            hash(s)

        elapsed = time.time() - t0
        print(elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_mutable_dict_not_hashable(self):
        with self.assertRaisesRegex(Exception, "not hashable"):
            hash(Dict(int, int)())

    def test_const_dict_str_perf(self):
        t = ConstDict(str, str)

        t0 = time.time()
        for i in range(100000):
            t({str(k): str(k+1) for k in range(10)})

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_const_dict_int_perf(self):
        t = ConstDict(int, int)

        t0 = time.time()
        for i in range(100000):
            t({k: k+1 for k in range(10)})

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_const_dict_iter_int(self):
        t = ConstDict(int, int)

        aDict = t({k: k+1 for k in range(100)})
        for k in aDict:
            self.assertEqual(aDict[k], k+1)

    def test_const_dict_iter_str(self):
        t = ConstDict(str, str)

        aDict = t({str(k): str(k+1) for k in range(100)})
        for k in aDict:
            self.assertEqual(aDict[str(k)], str(int(k)+1))

    def test_alternative_bytecounts(self):
        alt = Alternative(
            "Empty",
            X={},
            Y={}
        )

        self.assertEqual(_types.bytecount(alt), 1)
        self.assertEqual(_types.bytecount(alt.X), 1)
        self.assertEqual(_types.bytecount(alt.Y), 1)

    def test_alternatives_with_Bytes(self):
        alt = Alternative(
            "Alt",
            x_0={'a': bytes}
        )
        self.assertEqual(alt.x_0(a=b''), alt.x_0(a=b''))

    def test_alternatives_with_str_func(self):
        alt = Alternative(
            "Alt",
            x_0={'a': bytes},
            f=lambda self: 1,
            __str__=lambda self: "not_your_usual_str"
        )

        self.assertEqual(alt.x_0().f(), 1)
        self.assertEqual(str(alt.x_0()), "not_your_usual_str")

    def test_named_tuple_subclass_magic_methods(self):
        class X(NamedTuple(x=int, y=int)):
            def __str__(self):
                return "str override"

            def __repr__(self):
                return "repr override"

            def __call__(self):
                return "call implemented"

        self.assertEqual(repr(X()), "repr override")
        self.assertEqual(str(X()), "str override")
        self.assertEqual(X()(), "call implemented")

    def test_empty_alternatives(self):
        a = Alternative(
            "Alt",
            A={},
            B={}
        )

        self.assertEqual(a.A(), a.A())
        self.assertIsInstance(deserialize(a, serialize(a, a.A())), a.A)
        self.assertEqual(a.A(), deserialize(a, serialize(a, a.A())))

        self.assertEqual(a.B(), a.B())
        self.assertNotEqual(a.A(), a.B())
        self.assertNotEqual(a.B(), a.A())

    def test_extracted_alternatives_have_correct_type(self):
        Alt = Alternative(
            "Alt",
            A={},
            B={}
        )
        tOfAlt = TupleOf(Alt)

        a = Alt.A()
        aTup = tOfAlt((a,))

        self.assertEqual(a, aTup[0])
        self.assertTrue(type(a) is type(aTup[0]))  # noqa: F721

    def test_alternatives(self):
        alt = Alternative(
            "Alt",
            child_ints={'x': int, 'y': int},
            child_strings={'x': str, 'y': str}
        )

        self.assertTrue(issubclass(alt.child_ints, alt))
        self.assertTrue(issubclass(alt.child_strings, alt))

        a = alt.child_ints(x=10, y=20)
        a2 = alt.child_ints(x=10, y=20)

        self.assertEqual(a, a2)

        self.assertTrue(isinstance(a, alt))
        self.assertTrue(isinstance(a, alt.child_ints))

        self.assertEqual(a.x, 10)
        self.assertEqual(a.y, 20)
        self.assertTrue(a.matches.child_ints)
        self.assertFalse(a.matches.child_strings)

        with self.assertRaisesRegex(AttributeError, "immutable"):
            a.x = 20

    def test_alternatives_comparison(self):
        empty = Alternative("X", A={}, B={})

        self.assertEqual(empty.A(), empty.A())
        self.assertEqual(empty.B(), empty.B())
        self.assertNotEqual(empty.A(), empty.B())

        a = Alternative(
            "X",
            A={'a': int},
            B={'b': int},
            C={'c': str},
            D={'d': bytes},
        )

        self.assertEqual(a.A(a=10), a.A(a=10))
        self.assertNotEqual(a.A(a=10), a.A(a=11))

        self.assertNotEqual(a.C(c=""), a.C(c="hi"))
        self.assertFalse(a.C(c="") == a.C(c="hi"))
        self.assertNotEqual(a.D(d=b""), a.D(d=b"hi"))

    def test_alternatives_add_operator(self):
        alt = Alternative(
            "Alt",
            child_ints={'x': int, 'y': int},
            __add__=lambda lhs, rhs: (lhs, rhs)
        )

        a = alt.child_ints(x=0, y=2)

        self.assertEqual(a+a, (a, a))

    def test_alternatives_perf(self):
        alt = Alternative(
            "Alt",
            child_ints={'x': int, 'y': int},
            child_strings={'x': str, 'y': str}
        )

        t0 = time.time()

        for i in range(1000000):
            a = alt.child_ints(x=10, y=20)
            a.matches.child_ints
            a.x

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed, expected=2.0)

    def test_object_hashing_and_equality(self):
        for _ in range(100):
            producer = RandomValueProducer()
            producer.addEvenly(20, 2)

            values = producer.all()

            for v1 in values:
                for v2 in values:
                    if hash(v1) != hash(v2) and v1 == v2:
                        print(v1, v2, type(v1), type(v2))

            for v1 in values:
                for v2 in values:
                    if type(v1) == type(v2) and v1 == v2:
                        self.assertEqual(hash(v1), hash(v2), (v1, v2))
                        if type(v1) is type(v2):
                            self.assertEqual(repr(v1), repr(v2), (v1, v2, type(v1), type(v2)))

            values = sorted([makeTuple(v) for v in values])

            for i in range(len(values)-1):
                self.assertTrue(values[i] <= values[i+1])
                self.assertTrue(values[i+1] >= values[i])

    def test_bytes_repr(self):
        for _ in range(100000):
            # always start with a '"' because otherwise python keeps chosing different
            # initial characters.
            someBytes = b'"' + numpy.random.uniform(size=2).tostring()
            self.assertEqual(repr(makeTuple(someBytes)), repr((someBytes,)))

    def test_equality_with_native_python_objects(self):
        tups = [(1, 2, 3), (), ("2",), (b"2",), (1, 2, 3, "b"), (2,), (None,)]

        for tup1 in tups:
            self.assertEqual( makeTuple(*tup1), tup1 )

            for tup2 in tups:
                if tup1 != tup2:
                    self.assertNotEqual( makeTuple(*tup1), tup2 )

        for tup1 in tups:
            self.assertEqual( makeTupleOf(*tup1), tup1 )

            for tup2 in tups:
                if tup1 != tup2:
                    self.assertNotEqual( makeTupleOf(*tup1), tup2 )

    def test_add_tuple_of(self):
        tupleOfInt = TupleOf(int)

        tups = [(), (1, 2), (1,), (1, 2, 3, 4)]

        for tup1 in tups:
            for tup2 in tups:
                self.assertEqual(tupleOfInt(tup1) + tupleOfInt(tup2), tupleOfInt(tup1+tup2))
                self.assertEqual(tupleOfInt(tup1) + tup2, tupleOfInt(tup1+tup2))

    def test_slice_tuple_of(self):
        tupleOfInt = TupleOf(int)

        ints = tuple(range(20))
        aTuple = tupleOfInt(ints)

        for i in range(-21, 21):
            for i2 in range(-21, 21):
                for step in range(-3, 3):
                    if step != 0:
                        self.assertEqual(aTuple[i:i2:step], ints[i:i2:step])

            try:
                ints[i]
                self.assertEqual(aTuple[i], ints[i])
            except IndexError:
                with self.assertRaises(IndexError):
                    aTuple[i]

    def test_dictionary_subtraction_basic(self):
        intDict = ConstDict(int, int)

        self.assertEqual(intDict({1: 2}) - (1,), intDict({}))
        self.assertEqual(intDict({1: 2, 3: 4}) - (1,), intDict({3: 4}))
        self.assertEqual(intDict({1: 2, 3: 4}) - (3,), intDict({1: 2}))

    def test_dictionary_addition_and_subtraction(self):
        someDicts = [{i: choice([1, 2, 3, 4, 5]) for i in range(choice([4, 6, 10, 20]))} for _ in range(20)]
        intDict = ConstDict(int, int)

        for d1 in someDicts:
            for d2 in someDicts:
                addResult = dict(d1)
                addResult.update(d2)

                self.assertEqual(intDict(d1) + intDict(d2), intDict(addResult))

                res = intDict(addResult)

                while len(res):
                    toRemove = []

                    for i in range(choice(list(range(len(res))))+1):
                        key = choice(list(addResult))
                        del addResult[key]
                        toRemove.append(key)

                    res = res - toRemove

                    self.assertEqual(res, intDict(addResult))

    def test_subclassing(self):
        BaseTuple = NamedTuple(x=int, y=float)

        class NTSubclass(BaseTuple):
            def f(self):
                return self.x + self.y

            def __repr__(self):
                return "ASDF"

        inst = NTSubclass(x=10, y=20)

        self.assertTrue(isinstance(inst, BaseTuple))
        self.assertTrue(isinstance(inst, NTSubclass))
        self.assertTrue(type(inst) is NTSubclass)

        self.assertEqual(repr(inst), "ASDF")
        self.assertNotEqual(BaseTuple.__repr__(inst), "ASDF")

        self.assertEqual(inst.x, 10)
        self.assertEqual(inst.f(), 30)

        TupleOfSubclass = TupleOf(NTSubclass)

        instTup = TupleOfSubclass((inst, BaseTuple(x=20, y=20.0)))

        self.assertTrue(isinstance(instTup[0], NTSubclass))
        self.assertTrue(isinstance(instTup[1], NTSubclass))
        self.assertEqual(instTup[0].f(), 30)
        self.assertEqual(instTup[1].f(), 40)

        self.assertEqual(BaseTuple(inst).x, 10)

        self.assertTrue(OneOf(None, NTSubclass)(None) is None)
        self.assertTrue(OneOf(None, NTSubclass)(inst) == inst)

    def test_serialization_primitives(self):
        def checkCanSerialize(x):
            self.assertEqual(x, deserialize(type(x), serialize(type(x), x)), x)

        checkCanSerialize(0)
        checkCanSerialize(1)
        checkCanSerialize(2)
        checkCanSerialize(4)
        checkCanSerialize(8)
        checkCanSerialize(16)
        checkCanSerialize(32)
        checkCanSerialize(64)
        checkCanSerialize(128)
        checkCanSerialize(-1)
        checkCanSerialize(290)
        checkCanSerialize(1000)
        checkCanSerialize(99.5)
        checkCanSerialize("hi")
        checkCanSerialize(b"bye")
        checkCanSerialize(None)
        checkCanSerialize(True)
        checkCanSerialize(False)

    def test_serialization_bytecounts(self):
        ints = TupleOf(int)((1, 2, 3, 4))

        def varintBytecount(value):
            """the length (in bytes) of a varint"""
            res = 1
            while value >= 128:
                res += 1
                value /= 128
            return res

        while len(ints) < 1000000:
            ints = ints + ints
            t0 = time.time()

            expectedBytecount = (
                sum(varintBytecount(0) + varintBytecount(i) for i in ints) +
                (varintBytecount(0) * 3) + varintBytecount(len(ints))
            )

            self.assertEqual(len(serialize(TupleOf(int), ints)), expectedBytecount)

            print(time.time() - t0, " for ", len(ints))

    def test_serialization_roundtrip(self):
        for _ in range(100):
            producer = RandomValueProducer()
            producer.addEvenly(30, 3)

            values = producer.all()
            for v in values:
                ser = serialize(type(v), v)

                v2 = deserialize(type(v), ser)

                ser2 = serialize(type(v), v2)

                self.assertTrue(type(v2) is type(v))
                self.assertEqual(ser, ser2)
                self.assertEqual(str(v), str(v2))
                self.assertEqual(v, v2, (v, v2, type(v), type(v2), type(v) is type(v2)))

    def test_create_invalid_tuple(self):
        with self.assertRaises(TypeError):
            Tuple((int, int))

    def test_roundtrip_tuple(self):
        T = Tuple(str, bool, str)
        v = T(('1', False, ''))

        v2 = deserialize(T, serialize(T, v))

        self.assertEqual(v, v2)

    def test_roundtrip_alternative(self):
        A = Alternative("A", a0=dict(x_0=None))
        T = NamedTuple(x0=A, x1=bool)

        v = T(x0=A.a0(), x1=True)

        v2 = deserialize(T, serialize(T, v))

        self.assertEqual(v, v2)

    def test_serialize_doesnt_leak(self):
        T = TupleOf(int)

        def getMem():
            return psutil.Process().memory_info().rss / 1024 ** 2

        m0 = getMem()

        for passIx in range(100):
            for i in range(1000):
                t = T(list(range(i)))
                deserialize(T, serialize(T, t))

            self.assertTrue(getMem() < m0 + 100)

    def test_const_dict_of_tuple(self):
        K = NamedTuple(a=OneOf(float, int), b=OneOf(float, int))
        someKs = [K(a=0, b=0), K(a=1), K(a=10), K(b=10), K()]

        T = ConstDict(K, K)

        indexDict = {}
        x = T()

        numpy.random.seed(42)

        for _ in range(100):
            i1 = numpy.random.choice(len(someKs))
            i2 = numpy.random.choice(len(someKs))
            add = numpy.random.choice([False, True])

            if add:
                indexDict[i1] = i2
                x = x + {someKs[i1]: someKs[i2]}
            else:
                if i1 in indexDict:
                    del indexDict[i1]
                    x = x - (someKs[i1],)

            self.assertEqual(x, T({someKs[i]: someKs[v] for i, v in indexDict.items()}))
            for k in x:
                self.assertTrue(k in x)
                x[k]

    def test_conversion_of_binary_compatible(self):
        class T1(NamedTuple(a=int)):
            pass

        class T2(NamedTuple(a=int)):
            pass

        class T1Comp(NamedTuple(d=ConstDict(str, T1))):
            pass

        class T2Comp(NamedTuple(d=ConstDict(str, T1))):
            pass

        self.assertTrue(_types.isBinaryCompatible(T1Comp, T2Comp))
        self.assertTrue(_types.isBinaryCompatible(T1, T2))

    def test_binary_compatible_nested(self):
        def make():
            class Interior(NamedTuple(a=int)):
                pass

            class Exterior(NamedTuple(a=Interior)):
                pass

            return Exterior

        E1 = make()
        E2 = make()

        self.assertTrue(_types.isBinaryCompatible(E1, E2))

    def test_python_objects_in_tuples(self):
        class NormalPyClass(object):
            pass

        class NormalPySubclass(NormalPyClass):
            pass

        NT = NamedTuple(x=NormalPyClass, y=NormalPySubclass)

        nt = NT(x=NormalPyClass(), y=NormalPySubclass())
        self.assertIsInstance(nt.x, NormalPyClass)
        self.assertIsInstance(nt.y, NormalPySubclass)

    def test_construct_alternatives_with_positional_arguments(self):
        a = Alternative("A", HasOne={'a': str}, HasTwo={'a': str, 'b': str})

        with self.assertRaises(TypeError):
            a.HasTwo("hi")

        self.assertEqual(a.HasOne("hi"), a.HasOne(a="hi"))

        hasOne = a.HasOne("hi")
        self.assertEqual(a.HasOne(hasOne), hasOne)

        with self.assertRaises(TypeError):
            a.HasOne(a.HasTwo(a='1', b='b'))

    def test_recursive_classes_repr(self):
        A0 = Forward("A0")

        class ASelfRecursiveClass(Class):
            x = Member(OneOf(None, A0))

        A0 = A0.define(ASelfRecursiveClass)

        a = ASelfRecursiveClass()
        a.x = a

        b = ASelfRecursiveClass()
        b.x = b

        print(repr(a))

    def test_unsafe_pointers_to_list_internals(self):
        x = ListOf(int)()
        x.resize(100)
        for i in range(len(x)):
            x[i] = i

        aPointer = x.pointerUnsafe(0)
        self.assertTrue(str(aPointer).startswith("(Int64*)0x"))

        self.assertEqual(aPointer.get(), x[0])
        aPointer.set(100)
        self.assertEqual(aPointer.get(), 100)
        self.assertEqual(x[0], 100)

        aPointer = aPointer + 10

        self.assertEqual(aPointer.get(), x[10])
        self.assertEqual(aPointer[10], x[20])
        aPointer.set(20)
        self.assertEqual(aPointer.get(), 20)
        self.assertEqual(x[10], 20)

        # this is OK because ints are POD.
        aPointer.initialize(30)
        self.assertEqual(x[10], 30)

    def test_unsafe_pointers_to_uninitialized_list_items(self):
        # because this is testing unsafe operations, the test is
        # really just that we don't segfault!
        for _ in range(100):
            x = ListOf(TupleOf(int))()
            x.reserve(10)

            for i in range(x.reserved()):
                x.pointerUnsafe(i).initialize((i,))

            x.setSizeUnsafe(10)

        # now check that if we fail to set the size we'll leak the tuple
        aLeakedTuple = TupleOf(int)((1, 2, 3))
        x = ListOf(TupleOf(int))()
        x.reserve(1)
        x.pointerUnsafe(0).initialize(aLeakedTuple)
        x = None

        self.assertEqual(_types.refcount(aLeakedTuple), 2)

    def test_list_extend(self):
        LI = ListOf(int)
        LF = ListOf(float)

        li = LI([1, 2, 3])
        lf = LF([1.5, 2.5, 3.5])

        li.extend(li)
        self.assertEqual(li, [1, 2, 3, 1, 2, 3])

        lf.extend(lf)
        self.assertEqual(lf, [1.5, 2.5, 3.5, 1.5, 2.5, 3.5])

        lf.extend(li)
        self.assertEqual(lf, [1.5, 2.5, 3.5, 1.5, 2.5, 3.5, 1, 2, 3, 1, 2, 3])

        li = LI()
        li.extend(range(10))

        self.assertEqual(li, list(range(10)))

    def test_list_copy_operation_duplicates_list(self):
        T = ListOf(int)

        x = T([1, 2, 3])
        y = T(x)

        x[0] = 100

        self.assertNotEqual(y[0], 100)

    def test_list_and_tuple_conversion_to_numpy(self):
        for T in [ListOf(bool), TupleOf(bool)]:
            for arr in [
                    numpy.array([]),
                    numpy.array([0, 1, 2, 3, 4, 5]),
                    numpy.array([0, 1, 2, 3, 4, 5], 'int32'),
                    numpy.array([0, 1, 2, 3, 4, 5], 'int16'),
                    numpy.array([0, 1, 2, 3, 4, 5], 'bool')
            ]:
                self.assertEqual(T(arr), T(arr.tolist()))
                self.assertEqual(T(arr).toArray().tolist(), [bool(x) for x in arr.tolist()])

        for T in [ListOf(int), TupleOf(int)]:
            for arr in [
                    numpy.array([]),
                    numpy.array([1, 2, 3, 4, 5]),
                    numpy.array([1, 2, 3, 4, 5], 'int32'),
                    numpy.array([1, 2, 3, 4, 5], 'int16')
            ]:
                self.assertEqual(T(arr), T(arr.tolist()))
                self.assertEqual(T(arr).toArray().tolist(), arr.tolist())

        for T in [ListOf(float), TupleOf(float)]:
            for arr in [
                    numpy.array([]),
                    numpy.array([1, 2, 3, 4, 5], 'float64'),
                    numpy.array([1, 2, 3, 4, 5], 'float32')
            ]:
                self.assertEqual(T(arr), T(arr.tolist()))
                self.assertEqual(T(arr).toArray().tolist(), arr.tolist())

        self.assertEqual(str(ListOf(Int64)([1, 2, 3, 4]).toArray().dtype), 'int64')
        self.assertEqual(str(ListOf(Int32)([1, 2, 3, 4]).toArray().dtype), 'int32')
        self.assertEqual(str(ListOf(Int16)([1, 2, 3, 4]).toArray().dtype), 'int16')
        self.assertEqual(str(ListOf(Int8)([1, 2, 3, 4]).toArray().dtype), 'int8')

        self.assertEqual(str(ListOf(UInt64)([1, 2, 3, 4]).toArray().dtype), 'uint64')
        self.assertEqual(str(ListOf(UInt32)([1, 2, 3, 4]).toArray().dtype), 'uint32')
        self.assertEqual(str(ListOf(UInt16)([1, 2, 3, 4]).toArray().dtype), 'uint16')
        self.assertEqual(str(ListOf(UInt8)([1, 2, 3, 4]).toArray().dtype), 'uint8')

        self.assertEqual(str(ListOf(Float64)([1, 2, 3, 4]).toArray().dtype), 'float64')
        self.assertEqual(str(ListOf(Float32)([1, 2, 3, 4]).toArray().dtype), 'float32')

    def test_list_of_equality(self):
        x = ListOf(int)([1, 2, 3, 4])
        y = ListOf(int)([1, 2, 3, 5])

        self.assertEqual(x, x)
        self.assertNotEqual(x, y)

    def test_tuple_r_add(self):
        self.assertEqual(
            (1, 2, 4, 5, 6) + TupleOf(int)([1, 2]),
            (1, 2, 4, 5, 6, 1, 2)
        )

        self.assertEqual(
            [1, 2, 4, 5, 6] + TupleOf(int)([1, 2]),
            (1, 2, 4, 5, 6, 1, 2)
        )

        with self.assertRaises(TypeError):
            [1, 2, "hi", 5, 6] + TupleOf(int)([1, 2])

    def test_tuple_r_cmp(self):
        self.assertEqual(
            (1, 2, 3), TupleOf(int)([1, 2, 3])
        )

    def test_can_convert_numpy_scalars(self):
        self.assertEqual(OneOf(int, float)(numpy.int64(10)), 10)
        self.assertEqual(OneOf(int, float)(numpy.float64(10.5)), 10.5)

    def test_other_bitness_types(self):
        # verify we can cast around non-64-bit values in a way that matches numpy
        typeAndNumpyType = [
            (Bool, numpy.bool),
            (Int8, numpy.int8),
            (Int16, numpy.int16),
            (Int32, numpy.int32),
            (Int64, numpy.int64),
            (UInt8, numpy.uint8),
            (UInt16, numpy.uint16),
            (UInt32, numpy.uint32),
            (UInt64, numpy.uint64),
            (Float32, numpy.float32),
            (Float64, numpy.float64)
        ]

        for ourType, numpyType in typeAndNumpyType:
            for candValue in [-1, 0, 1, 10, 100, 1000, 100000, 10000000, 10000000000]:
                self.assertEqual(int(ourType(candValue)), int(numpyType(candValue)), (ourType, candValue))
                self.assertEqual(float(ourType(candValue)), float(numpyType(candValue)), (ourType, candValue))

            for ourType2, numpyType2 in typeAndNumpyType:
                zeroOrTwoFloatTypes = sum([1 if 'float' in str(t) else 0 for t in [numpyType, numpyType2]]) in [0, 2]

                if zeroOrTwoFloatTypes:
                    for candValue in [-1, 0, 1, 10, 100, 1000, 100000, 10000000, 10000000000]:
                        self.assertEqual(
                            int(ourType(ourType2(candValue))),
                            int(numpyType(numpyType2(candValue))),
                            (ourType, ourType2, candValue)
                        )
                        self.assertEqual(
                            float(ourType(ourType2(candValue))),
                            float(numpyType(numpyType2(candValue))),
                            (ourType, ourType2, candValue)
                        )
                else:
                    # we convert from float to int as c++, which is different than numpy, which clips
                    # floats in a bizarre way. e.g.
                    #  numpy.int16(numpy.int64(numpy.float64(10000000000)))
                    # is not
                    #  numpy.int16(numpy.float64(10000000000))
                    pass

    def test_other_bitness_types_operators(self):

        def add(x, y):
            return x+y

        def div(x, y):
            return x/y

        def mul(x, y):
            return x*y

        def sub(x, y):
            return x-y

        def bitand(x, y):
            return x&y

        def bitor(x, y):
            return x|y

        def bitxor(x, y):
            return x^y

        otherTypes = [Bool, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float32, Float64]
        for t1 in otherTypes:
            for t2 in otherTypes:
                for op in [add, mul, div, sub, bitand, bitor, bitxor]:
                    if not ((t1.IsFloat or t2.IsFloat) and op in (bitand, bitor, bitxor)):
                        res = op(t1(10), t2(10))
                        resType = type(res)
                        resType = {bool: Bool, int: Int64, float: Float64}.get(resType, resType)

                        if t1.IsFloat and t2.IsFloat:
                            self.assertTrue(resType.IsFloat)
                            self.assertEqual(resType.Bits, max(t1.Bits, t2.Bits))
                            self.assertEqual(res, op(10, 10))
                        elif t1.IsFloat or t2.IsFloat:
                            self.assertTrue(resType.IsFloat)
                            self.assertEqual(resType.Bits, t1.Bits if t1.IsFloat else t2.Bits)
                            if t1.Bits > 1 and t2.Bits > 1:
                                self.assertEqual(res, op(10, 10))
                        elif t1 is Bool and t2 is Bool:
                            self.assertEqual(resType, Bool if op in (bitor, bitand, bitxor) else Int64 if op is not div else Float64)
                        else:
                            self.assertEqual(resType.Bits, max(t1.Bits, t2.Bits))

                            if op is not div:
                                self.assertEqual(resType.IsSignedInt, t1.IsSignedInt or t2.IsSignedInt)

                            if t1.Bits > 1 and t2.Bits > 1:
                                self.assertEqual(res, op(10, 10))

    def test_comparing_arbitrary_objects(self):
        x = TupleOf(object)(["a"])
        y = TupleOf(object)([1])

        with self.assertRaises(TypeError):
            x < y

        self.assertEqual(x, x)
        self.assertEqual(y, y)
        self.assertNotEqual(x, y)

    def test_list_of_indexing_with_numpy_ints(self):
        x = ListOf(ListOf(int))([[1, 2, 3], [4, 5, 6]])
        self.assertEqual(x[numpy.int64(0)][numpy.int64(0)], 1)

    def test_dispatch_tries_without_conversion_first(self):
        class ClassWithForcedConversion(Class):
            def f(self, x: float):
                return "float"

        class ClassWithBoth(Class):
            def f(self, x: float):
                return "float"

            def f(self, x: int):  # noqa: F811
                return "int"

            def f(self, x: bool):  # noqa: F811
                return "bool"

        # swap the order
        class ClassWithBoth2(Class):
            def f(self, x: bool):
                return "bool"

            def f(self, x: int):  # noqa: F811
                return "int"

            def f(self, x: float):  # noqa: F811
                return "float"

        self.assertEqual(ClassWithForcedConversion().f(10), "float")
        self.assertEqual(ClassWithForcedConversion().f(10.5), "float")
        self.assertEqual(ClassWithForcedConversion().f(True), "float")

        self.assertEqual(ClassWithBoth().f(10), "int")
        self.assertEqual(ClassWithBoth().f(10.5), "float")
        self.assertEqual(ClassWithBoth().f(True), "bool")

        self.assertEqual(ClassWithBoth2().f(10), "int")
        self.assertEqual(ClassWithBoth2().f(10.5), "float")
        self.assertEqual(ClassWithBoth2().f(True), "bool")

    def test_error_message_on_bad_dispatch(self):
        @Function
        def f(x: int):
            return x

        with self.assertRaisesRegex(TypeError, "str"):
            f("hi")

        with self.assertRaisesRegex(TypeError, "argname="):
            f(argname=1)

    def test_named_tuple_comparison(self):
        N = NamedTuple(x=OneOf(None, int), y=OneOf(None, int))

        class S(N):
            pass

        self.assertEqual(N(x=1, y=2), N(x=1, y=2))
        self.assertNotEqual(N(x=1, y=2), N(x=1, y=3))
        self.assertFalse(N(x=1, y=2) == N(x=1, y=3))

        self.assertEqual(S(x=1, y=2), S(x=1, y=2))
        self.assertNotEqual(S(x=1, y=2), S(x=1, y=3))
        self.assertFalse(S(x=1, y=2) == S(x=1, y=3))

    def test_const_dict_comparison_more(self):
        N = NamedTuple(x=OneOf(None, int), y=OneOf(None, int))
        D = ConstDict(str, N)

        n1 = N(x=1, y=2)
        n2 = N(x=1, y=3)

        self.assertEqual(D({'a': n1}), D({'a': n1}))
        self.assertNotEqual(D({'a': n1}), D({'a': n2}))

    def test_mutable_dict(self):
        T = Dict(int, int)

        d = T()

        self.assertEqual(len(d), 0)

        with self.assertRaises(KeyError):
            d[0]

        d[0] = 10

        self.assertEqual(len(d), 1)
        self.assertEqual(d[0], 10)

        d[0] = 20

        self.assertEqual(len(d), 1)
        self.assertEqual(d[0], 20)

        for i in range(2000):
            d[i] = i

            if i % 100 == 0 and i:
                for x in range(len(d)):
                    self.assertEqual(d[x], x)

        for i in range(2000):
            if i % 100 == 0 and i:
                for x in range(i):
                    assert x not in d

                for x in range(i, 2000):
                    self.assertEqual(d[x], x)

            del d[i]

        # verify that adding and removing elements doesn't leak memory
        usage = currentMemUsageMb()
        for i in range(1000000):
            d[0] = i
            del d[0]
        self.assertLess(currentMemUsageMb(), usage+1)

    def test_mutable_dict_fuzz(self):
        native_d = Dict(int, int)()
        py_d = {}

        for dictSize in [10, 100, 1000, 10000]:
            for i in range(100000):
                z = numpy.random.choice(dictSize)

                self.assertEqual(z in py_d, z in native_d)

                if i % 3 == 0 or (i % 1000) > 900:
                    if z in py_d:
                        del py_d[z]
                        del native_d[z]
                else:
                    py_d[z] = i
                    native_d[z] = i

            for i in range(dictSize):
                self.assertEqual(z in py_d, z in native_d)

    def test_mutable_dict_refcounts(self):
        native_d = Dict(str, ListOf(int))()
        i = ListOf(int)()

        native_d["a"] = i

        self.assertEqual(_types.refcount(i), 2)

        native_d["b"] = i

        self.assertEqual(_types.refcount(i), 3)

        del native_d["a"]

        self.assertEqual(_types.refcount(i), 2)

        native_d["b"] = ListOf(int)()

        self.assertEqual(_types.refcount(i), 1)

        native_d["b"] = i

        self.assertEqual(_types.refcount(i), 2)

        native_d = None

        self.assertEqual(_types.refcount(i), 1)

    def test_mutable_dict_create_many(self):
        for ct in range(100):
            d = Dict(int, int)()
            for i in range(ct):
                d[i] = i + 1

    def test_mutable_dict_methods(self):
        d = Dict(int, int)({i: i+1 for i in range(10)})

        self.assertEqual(list(d.keys()), list(range(10)))
        self.assertEqual(list(d.values()), list(range(1, 11)))
        self.assertEqual(list(d.items()), [(i, i+1) for i in range(10)])

        for i in range(10):
            self.assertEqual(d.get(i), i+1)
            self.assertEqual(d.get(i, None), i+1)

        self.assertEqual(d.get(1000), None)
        self.assertEqual(d.get(1000, 123), 123)

        with self.assertRaises(TypeError):
            self.assertEqual(d.get("1000"), None)

    def test_mutable_dict_iteration_order(self):
        d = Dict(int, int)()

        d[10] = 10
        d[1] = 1
        d[2] = 2

        self.assertEqual(list(d), [10, 1, 2])
        del d[1]
        self.assertEqual(list(d), [10, 2])

    def test_simplicity(self):
        isSimple = _types.isSimple

        self.assertTrue(isSimple(int))
        self.assertTrue(isSimple(Int32()))
        self.assertTrue(isSimple(Int16()))
        self.assertTrue(isSimple(Int8()))
        self.assertTrue(isSimple(UInt64()))
        self.assertTrue(isSimple(UInt32()))
        self.assertTrue(isSimple(UInt16()))
        self.assertTrue(isSimple(UInt8()))
        self.assertTrue(isSimple(str))
        self.assertTrue(isSimple(bytes))
        self.assertTrue(isSimple(bool))
        self.assertTrue(isSimple(float))

        class C(Class):
            pass

        self.assertFalse(isSimple(C))

        self.assertTrue(isSimple(ListOf(int)))
        self.assertFalse(isSimple(ListOf(C)))

        self.assertTrue(isSimple(TupleOf(int)))
        self.assertFalse(isSimple(TupleOf(C)))

        self.assertTrue(isSimple(ConstDict(int, int)))
        self.assertFalse(isSimple(ConstDict(C, int)))
        self.assertFalse(isSimple(ConstDict(int, C)))

        self.assertTrue(isSimple(Dict(int, int)))
        self.assertFalse(isSimple(Dict(C, int)))
        self.assertFalse(isSimple(Dict(int, C)))

        self.assertFalse(isSimple(Alternative("Alternative")))

        self.assertTrue(isSimple(NamedTuple(x=int)))
        self.assertFalse(isSimple(NamedTuple(x=C)))

        X = Forward("X")
        X = X.define(Alternative("X", X={'x': X}, Y={'i': int}))
        self.assertFalse(isSimple(X))
        self.assertFalse(isSimple(NamedTuple(x=X)))

        self.assertFalse(isSimple(OneOf(int, X)))
        self.assertTrue(isSimple(OneOf(int, float)))

    def test_oneof_picks_best_choice(self):
        T = OneOf(float, int, bool)

        self.assertIsInstance(T(1.5), float)
        self.assertIsInstance(T(1), int)
        self.assertIsInstance(T(True), bool)


# -*- coding: utf-8 -*-
import re
from newscrawl.items import newsItem
from scrapy_redis.spiders import RedisCrawlSpider
from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.http import Request

class ShenZhenDailySpider(RedisCrawlSpider):
    name = "shenzhendaily"
    allowed_domains = ["sznews.com"]
    base_url = "http://sztqb.sznews.com/"
    newspapers = "深圳特区报"
    redis_key = "shenzhendaily:start_urls"

    def parse_start_url(self, response):
        areas = response.xpath('//div[@class="Chunkiconlist"]/p/a[1]')
        for area in areas:
            page = area.xpath('@href').extract()[0]
            url = re.sub('\w{1,}.html',page,response.url)
            str_category = area.xpath('text()').extract()[0]
            category = str_category.split('：')[1]
            yield Request(url, self.page_parse, dont_filter=True, meta={'category':category})

    def page_parse(self, response):
        articles = response.xpath('//div[@class="newslist"]/ul/li/h3/a/@href').extract()
        category = response.meta['category']
        for article in articles:
            article_path = re.findall('/\w{1,}/\w{1,}/\w{1,}/\w{1,}.html', article)[0]
            url = self.base_url + 'PC' + article_path
            yield Request(url, self.parse_item, meta={'category':category})

    def parse_item(self, response):
        list_title = response.xpath('//div[@class="newsdetatit"]/h3/text()').extract()
        title = "".join(list_title).strip()
        list_content = response.xpath('//div[@class="newsdetatext"]/founder-content/p/text()').extract()
        content = "".join(list_content).strip()
        list_date = re.findall('(?<=/)\d{1,}/\d{1,}(?=/)', response.url)
        str_date = "".join(list_date)
        n_date = str_date.replace('/','-')
        date = n_date[:4] + '-' + n_date[4:]
        list_page = response.xpath('//div[@class="newsdetatit"]/p[3]/span[@class="Author"]/text()').extract()
        str_page = "".join(list_page)
        page = str_page.split('：')[1]
        if content == "":
            pass
        else:
            item = newsItem()
            item['title'] = title
            item['page'] = page
            item['content'] = content
            item['date'] = date
            item['category'] = response.meta['category']
            item['url'] = response.url
            item['newspapers'] = self.newspapers
            yield item


from django.utils import timezone
from media_management_api.media_service.models import UserProfile, Course, CourseUser
from .models import Application, Token
from .exceptions import InvalidApplicationError, InvalidTokenError

import datetime
import logging

logger = logging.getLogger(__name__)

# Expiration time for tokens. Dictionary keys are intended
# to match the arguments accepted by datetime.timedelta():
#       https://docs.python.org/2/library/datetime.html#datetime.timedelta
TOKEN_EXPIRE = {"hours": 240}

# Token refresh time used when obtaining recent tokens to ensure that the token is good for _at least_ 24 hours
TOKEN_REFRESH = {"hours": 240 - 24}

def obtain_token(data):
    # Check that the required data are present
    required_data = ('client_id', 'client_secret', 'user_id')
    if not all([k in data for k in required_data]):
        raise InvalidApplicationError("Missing required data. Must include: %s" % ", ".join(required_data))

    # Validate the application
    application = None
    if is_valid_application(client_id=data['client_id'], client_secret=data['client_secret'], raise_exception=True):
        application = Application.objects.get(client_id=data['client_id'], client_secret=data['client_secret'])

     # Get or create a user profile
    if not data['user_id']:
        raise InvalidTokenError("Invalid user_id - must not be empty")
    user = get_or_create_user(data['user_id'])

    # Add the user to the specified course (this information is not saved in the token)
    course_id = data.get("course_id", None)
    course_permission = None
    if course_id:
        course_permission = data.get('course_permission', 'read')
        course_user = add_user_to_course(user=user, course_id=data['course_id'], is_admin=course_permission == 'write')

    # Try to reuse the most recent token for the user if it's still valid, otherwise
    # create a new token.
    token_attrs = {
        "user_profile": user.profile,
        "application": application,
    }
    recent_tokens = Token.objects.filter(**token_attrs).order_by('-created')[0:1]
    if len(recent_tokens) == 0 or is_token_expired(recent_tokens[0]) or is_token_refreshable(recent_tokens[0]):
        token = Token(**token_attrs)
        token.save()
    else:
        token = recent_tokens[0]

    # Return the token info
    token_expiration = get_token_expiration(token)
    token_response = {
        "user_id": data["user_id"],
        "access_token": token.key,
        "expires": token_expiration.strftime("%Y-%m-%d %H:%M:%S"),
        "course_id": course_id,
        "course_permission": course_permission
    }
    logger.debug("Obtained token: %s" % token_response)
    return token_response

def get_token(access_token):
    return Token.objects.get(key=access_token)

def get_token_expiration(token):
    return token.created + datetime.timedelta(**TOKEN_EXPIRE)

def get_token_refresh(token):
    return token.created + datetime.timedelta(**TOKEN_REFRESH)

def get_or_create_user(user_id):
    user_profile = UserProfile.get_or_create_profile(user_id)
    return user_profile.user

def add_user_to_course(user=None, course_id=None, is_admin=False):
    try:
        course_user = CourseUser.add_user_to_course(user=user, course_id=course_id, is_admin=is_admin)
    except Course.DoesNotExist:
        raise InvalidTokenError("Course '%s' not found" % course_id)
    return course_user

def get_access_token_from_request(request):
    authorization = request.META.get('HTTP_AUTHORIZATION', '')
    access_token = None
    if authorization.lower().startswith("token "):
        access_token = authorization.split(" ", 2)[1]
    return access_token

def assert_token_valid(token):
    return is_token_valid(token, raise_exception=True)

def is_token_valid(token, raise_exception=False):
    return not is_token_expired(token, raise_exception=raise_exception)

def is_token_expired(token, raise_exception=False):
    if not isinstance(token, Token):
        try:
            token = Token.objects.get(key=token)
        except Token.DoesNotExist:
            if raise_exception:
                raise InvalidTokenError("Invalid token")
            return True

    now = timezone.now()
    expiration = get_token_expiration(token)
    if now > expiration:
        if raise_exception:
            logger.debug("Token %s expired at %s" % (token.pk, expiration))
            raise InvalidTokenError("Token has expired")
        return True
    return False

def is_token_refreshable(token):
    now = timezone.now()
    refresh = get_token_refresh(token)
    return now > refresh

def is_valid_application(client_id=None, client_secret=None, raise_exception=False):
    try:
        application = Application.objects.get(client_id=client_id, client_secret=client_secret)
    except Application.DoesNotExist:
        if raise_exception:
            raise InvalidApplicationError("Invalid application.")
        return False
    return True

def destroy_token(access_token):
    try:
        Token.objects.get(key=access_token).delete()
    except:
        return True
    return True


"""
134. LRU Cache
https://www.lintcode.com/problem/lru-cache/description

整体思路：
首先数据结构需要不断删除然后添加到末尾。并且需要有序。用linkedlist比较适合增加删除操作。但是删除本身如果要能在O(1)时间内实现还需要hashmap辅助，用于记录需要删除节点的前面的节点。
"""
class LinkedNode:
    def __init__(self, key=None, value=None, next=None):
        self.key = key
        self.value = value
        self.next = next

class LRUCache:
    """
    @param: capacity: An integer
    """
    def __init__(self, capacity):
        # do intialization if necessary
        self.key_to_prev_node = {}
        self.dummy = LinkedNode()
        self.tail = self.dummy
        self.capacity = capacity

    """
    @param: key: An integer
    @return: An integer
    """
    def get(self, key):
        # write your code here
        if key not in self.key_to_prev_node:
            return -1
        prev_node = self.key_to_prev_node[key]
        node = prev_node.next
        self.kick_to_back(prev_node)
        return node.value

    def kick_to_back(self, prev_node):
        current_node = prev_node.next
        if current_node == self.tail:
            return
        self.key_to_prev_node[current_node.next.key] = prev_node
        self.key_to_prev_node[current_node.key] = self.tail

        prev_node.next = current_node.next
        self.tail.next = current_node
        self.tail = current_node
        self.tail.next = None

    """
    @param: key: An integer
    @param: value: An integer
    @return: nothing
    """
    def set(self, key, value):
        # write your code here
        if key in self.key_to_prev_node:
            prev_node = self.key_to_prev_node[key]
            prev_node.next.value = value
            self.kick_to_back(prev_node)
        else:
            self.key_to_prev_node[key] = self.tail
            self.tail.next = LinkedNode(key, value)
            self.tail = self.tail.next
            if len(self.key_to_prev_node) > self.capacity:
                self.remove_oldest_node()

    def remove_oldest_node(self):
        if not self.dummy.next:
            return
        head = self.dummy.next
        if not head.next:
            self.dummy.next = None
            self.tail = self.dummy
        else:
            self.key_to_prev_node[head.next.key] = self.dummy
            self.dummy.next = head.next
        del self.key_to_prev_node[head.key]


import struct
# from src import summary_pb2
import event_pb2
# from crc32c import crc32c
import io
with open('demo.pb', 'rb') as f:
    data = f.read()

def read(data):
    header = struct.unpack('Q', data[:8])
    # print('header:', header)
    # print('expected header crc:',masked_crc32c(header))
    # print(header)
    data = data[8:]
    # print(data[:4])

    crc_hdr = struct.unpack('I', data[:4])
    # print('read header crc:',crc_hdr)
    # print([hex(i) for i in crc_hdr])
    data = data[4:]

    event_str = data[:int(header[0])]
    # print([hex(i) for i in event_str])
    data = data[int(header[0]):]

    crc_ev = struct.unpack('>I', data[:4])
    # print([hex(i) for i in crc_ev])
    data = data[4:]

    return data, event_str
    # print(len(event_str))
    summ = event_pb2.Event()
    # summ = summary_pb2.Summary()
    summ.ParseFromString(event_str)
    print(summ)
    return data

def save_img(encoded, step):
    from PIL import Image
    img = Image.open(io.BytesIO(encoded))
    img.save('img_{}.png'.format(step), format='png')

while data:
    data, event_str = read(data)
    event = event_pb2.Event()

    # summ = summary_pb2.Summary()
    event.ParseFromString(event_str)
    if event.HasField('summary'):
        for value in event.summary.value:
            if value.HasField('image'):
                img = value.image
                save_img(img.encoded_image_string, event.step)
                print('img')

    # print(event)


import math
import os
import random
import re
import sys

# Complete the formingMagicSquare function below.
#def formingMagicSquare(s):

def anagram(s1,s2):
    if( len(s1) != len(s2)):
        print("false")
        return
    for x in range(len(s2)):
        if(s1[x] != s2[x]):
            print("false")
            return
    print("true")

s1 = input()
s1 = list(s1)

s2 = input()
s2 = list(s2)

s1.sort()
s2.sort()
anagram(s1,s2)


from django.shortcuts import render, get_object_or_404

from django.http import HttpResponse

from .models import StudySpace

def index(request):
    """
    The index page should show a list of study spaces available across all
    departments
    """
    study_space_list = StudySpace.objects.order_by('-department')
    context = {'study_space_list': study_space_list}
    return render(request, 'StudySpaceFinderApp/index.html', context)

def study_space_page(request, slug):
    """
    The Study Space Page should show some more information about a
    specific study space
    """
    study_space = get_object_or_404(StudySpace, slug=slug)
    return render(request, 'StudySpaceFinderApp/study_space_detail.html', {'study_space': study_space})


import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
import sqlalchemy as sa
from sqlalchemy_utils import database_exists, create_database
from datetime import datetime as dt

class DbSqlA:
    base        = declarative_base()
    def __init__(self,ConnectionString,fast_executemany=True):
        self.engine                 = sa.create_engine(ConnectionString) if not fast_executemany else sa.create_engine(ConnectionString,fast_executemany=True)
        self.session                = scoped_session(sessionmaker(autocommit=False,autoflush=False,bind=self.engine))
        self.base.query             = self.session.query_property()
        self.orm_session            = orm.scoped_session(orm.sessionmaker())(bind=self.engine)
        self.base.metadata.bind     = self.engine

    def create_all(self):
        if not database_exists(self.engine.url):
            create_database(self.engine.url)
        self.base.metadata.create_all(self.engine)

def tblUts(db):
    class auxDB:
        id      = sa.Column(sa.Integer,primary_key=True,autoincrement=True)

        def save(self):
            try:
                if not self.id:
                    db.session.add(self)
                db.session.commit()
            except:
                db.session.rollback()
                raise

        def delete(self):
            db.session.rollback()
            db.session.delete(self)
            db.session.commit()
    return auxDB


import pyrow
import time
from Tkinter import *
import datetime

restart = True

time_start = time.time()#declaration

fill1, fill2 = "maroon", "black"
fnt_msv = ('Helvetica', 110)
fnt_lrg = ('Helvetica', 65)
fnt_sml = ('Helvetica', 28)
fnt_colour = "white"
border = 20

distance = 100

ergs = list(pyrow.find())
if len(ergs) == 0:
	print("No ergs found.")
else:
	print("Connected to %d ergs" % len(ergs))

def printableTimeHMS(seconds):
	return time.strftime("%H:%M:%S", time.gmtime(seconds))

def printableTimeMS(seconds):
	return time.strftime("%M:%S", time.gmtime(seconds))

def drawScreen(canvas, width, height, meters_left, time_elapsed, time_projected, current_split, average_split, stroke_rate):
	#draw rectangles
	canvas.delete(ALL)
	canvas.create_rectangle(0, 0, width, height, fill = fill1) #background
	canvas.create_rectangle(border, border, width - border, (height - border)/2.5, fill = fill2) #top

	canvas.create_rectangle(border, (height + border)/2.5, (width - border)/2, height - border, fill = fill2) #left_bottom
	canvas.create_rectangle((width + border)/2, (height + border)/2.5, width - border, height - border, fill = fill2) #right_bottom
	#draw text

	digits = len(str(meters_left))
	canvas.create_text(width/2, height/6, anchor = CENTER, text = str(format(meters_left, str(digits) + ",d")), font = fnt_msv, fill = fnt_colour)
	canvas.create_text(0.7 * width,  0.29 * height, anchor = CENTER, text = "meters remaining", font = fnt_sml, fill = fnt_colour)

	canvas.create_text(width/4, 0.6 * height, anchor = CENTER, text = printableTimeHMS(time_elapsed), font = fnt_lrg, fill = fnt_colour)
	canvas.create_text(0.3 * width,  0.69 * height, anchor = CENTER, text = "time elapsed", font = fnt_sml, fill = fnt_colour)

	canvas.create_text(width/4, 0.8 * height, anchor = CENTER, text = printableTimeHMS(time_projected), font = fnt_lrg, fill = fnt_colour)
	canvas.create_text(0.3 * width,  0.89 * height, anchor = CENTER, text = "time projected", font = fnt_sml, fill = fnt_colour)

	canvas.create_text(3 * width/4, 0.8 * height, anchor = CENTER, text = printableTimeMS(current_split), font = fnt_lrg, fill = fnt_colour)
	canvas.create_text(0.8 * width,  0.89 * height, anchor = CENTER, text = "/500m", font = fnt_sml, fill = fnt_colour)

	canvas.create_text(0.84 * width, 0.1 * height, anchor = CENTER, text = str(stroke_rate), font = fnt_lrg, fill = fnt_colour)
	canvas.create_text(0.95 * width,  0.12 * height, anchor = CENTER, text = "s/m", font = fnt_sml, fill = fnt_colour)

	canvas.create_text(3 * width/4, 0.6 * height, anchor = CENTER, text = printableTimeMS(average_split), font = fnt_lrg, fill = fnt_colour)
	canvas.create_text(0.8 * width,  0.69 * height, anchor = CENTER, text = "average /500m", font = fnt_sml, fill = fnt_colour)

def updateScreen(canvas, width, height):

	meters_left = distance
	speed = 0
	rate = 0
	for x in range(0, len(ergs)):
		erg_object = pyrow.pyrow(ergs[x])
		monitor = erg_object.get_monitor()
		meters_left -= monitor['distance']
		split = monitor['pace']
		if (split != 0):
			speed += 1/split
		rate += monitor['spm']
	global restart
	if (restart and meters_left < distance):
		restart = False
		global time_start
		time_start = time.time()

	if meters_left < 0:
		meters_left = 0

	#print ("Meters Left: ", meters_left)
	#print ("Ergs connected: ", len(list(pyrow.find())))

	elapsed_time = 0 if restart else time.time() - time_start

	if (meters_left < distance):
		split = 0 if speed == 0 else 1/speed
		v = 1 if elapsed_time == 0 else (distance - meters_left) / elapsed_time
		average_split = 500 / v
		time_projected = elapsed_time + meters_left * split / 500
	else:
		split = 0
		average_split = 0
		time_projected = 0

	drawScreen(canvas, width, height, int(meters_left), elapsed_time, time_projected, split, average_split ,rate)
	return meters_left

def updateScreenWrapper(canvas, width, height):
	canvas.delete(ALL)
	meters_left = updateScreen(canvas, width, height)
	if (meters_left > 0):
		canvas.after(250, updateScreenWrapper, canvas, width, height)

def window():
	root = Tk()
	width, height = root.winfo_screenwidth(), root.winfo_screenheight() - 80
	canvas = Canvas(root, width=width, height=height)
	canvas.pack()

	if (len(ergs) != 0): updateScreenWrapper(canvas, width, height)
	else: drawScreen(canvas, width, height, 567322, 2473, 2314, 124, 118, 76)
	#distance_countdown(1000000, canvas)

	root.mainloop()  # blocks until window is closed

window()


import discord
from discord.ext import commands
import threading
import os
from random import shuffle, choice
from cogs.utils.dataIO import dataIO
from cogs.utils import checks
from cogs.utils.chat_formatting import pagify, escape
from urllib.parse import urlparse
from __main__ import send_cmd_help, settings
from json import JSONDecodeError
import re
import logging
import collections
import copy
import asyncio
import math
import time
import inspect
import subprocess
import urllib.parse

#IMPORTANT ?
import youtube_dl

youtube_dl_options = {
    'source_address': '0.0.0.0',
    'format': 'bestaudio/best',
    'extractaudio': True,
    'audioformat':'mp3',
    'nocheckcertificate':True,
    'ignoreerrors':False,
    'quiet':True,
    'no_warnings':True,
    'outtmpl':'data/audio/shared/%(id)s',
    'default_search':'auto',
    'encoding':'utf-8'
}

def display_time(seconds):
    return "**{}:{}**".format(seconds//60, seconds%60)

class Maudio:
    def __init__(self, bot):
        self.bot = bot
        self.playlist = {}
        self.downloaders = []
        self.bot_players = []
        self.settings = dataIO.load_json("data/audio/shared/settings.json")
        self.bot_players.append(self.bot)
        self.playlist = {self.bot:[]}
        self.skip_votes = {self.bot:[]}
        #self.iron = self.bot.get_cog("Iron")
        #self.silver = self.bot.get_cog("Silver")
        #self.gold = self.bot.get_cog("Gold")
        #self.platinum = self.bot.get_cog("Platinum")
        #try:
        #    self.add_all_bots()
        #except:
        #    pass
        self.tempshit = False
    def add_all_bots(self):
        self.bot_players.append(self.iron.bot)
        self.bot_players.append(self.silver.bot)
        self.bot_players.append(self.gold.bot)
        self.bot_players.append(self.platinum.bot)
        self.playlist = {
            self.iron: [],
            self.silver: [],
            self.gold: [],
            self.platinum: [],
            self.bot:[]
        }

    @commands.command(pass_context=True)
    async def set_vol(self, ctx, volume:int=None):
        """The command to set the volume

        It can go up to 200 but at 200 you should expect clipping."""
        if not volume:
            await self.bot.say("Current volume : {}".format(self.settings["VOLUME"]))
            return
        if volume > 200 or volume < 0:
            await self.bot.say("Invalid volume.")
        else:
            volume/=100
            self.settings["VOLUME"] = volume
            for i in self.bot_players:
                i.voice_client_in(ctx.message.server).audio_player.volume = volume
    @commands.command(pass_context=True)
    @checks.is_owner()
    async def set_bots(self, ctx):
        try:
            self.add_all_bots()
        except:
            await self.bot.say("There was an error while adding bots. Don't know "
                "what happened and my creator is kind of lazy so git gud bitsh.")
    @commands.command(pass_context=True)
    @checks.is_owner()
    async def dc_all(self, ctx):

        for player in self.bot_players:
            if player == self.bot:
                continue
            await player.send_message(ctx.message.channel, "Disconnecting {}".format(player.user.name))

            await player.logout()
            del player
    async def _join_voice_channel(self, player, channel):
        try:
            await asyncio.wait_for(player.join_voice_channel(channel), timeout=5, loop=player.loop)
        except asyncio.futures.TimeoutError:
            await self.bot.say("Bot failed to connect to voice channel, try again in 10 mins.")

    async def _create_ffmpeg_player(self, player,channel, song, start_time=None, end_time=None):
        server = channel.server
        voice_channel = self.playlist[player][0][0]
        voice_client = player.voice_client_in(server)
        if voice_client is None:
            to_connect = voice_channel
            if to_connect is None:
                #raise VoiceNotConnected("Okay somehow we're not connected and "
                #                        "we have no valid channel to "
                #                        "reconnect. In other words...LOL "
                #                        "REKT.")
                pass
            await self._join_voice_channel(player, to_connect)
            voice_client = player.voice_client_in(server)

        if voice_client.channel != voice_channel:
            await self._join_voice_channel(player, voice_channel)

        file_name = os.path.join("data/audio/shared", song.id)
        use_avconv = self.settings["AVCONV"]
        options = '-b:a 64k -bufsize 64k'
        before_options = ""
        if start_time:
            before_options += '-ss {}'.format(start_time)
        if end_time:
            options +=" -to {} -copyts".format(end_time)

        try:
            voice_client.audio_player.process.kill()
        except AttributeError:
            pass
        except ProcessLookupError:
            pass

        voice_client.audio_player = voice_client.create_ffmpeg_player(file_name, use_avconv=use_avconv, options=options, before_options=before_options)

        vol = self.settings["VOLUME"] / 100
        voice_client.audio_player.volume = vol

        return voice_client

    def garantee_bot(self,song,channel):
        player = None
        #player = discord.utils.find(lambda x: x.voice_client_in(channel.server).channel == channel, self.bot_players)
        for i in self.bot_players:
            if i.user in channel.voice_members:
                player = i
                break

        if player:
            if len(self.playlist[player])>=4:
                return
            return player

        player = discord.utils.find(lambda x: self.playlist[x] == [], self.bot_players)
        if player:
            return player

        ls = []
        for x in self.bot_players:
            ls.append((sum(j.duration for i,j in self.playlist[x]),x))
        return min(ls,key=lambda y: y[0])[1]

    @commands.command(pass_context=True, no_pm=True)
    async def adl(self, ctx, *, url_or_search_terms):
        url = url_or_search_terms
        server = ctx.message.server
        author = ctx.message.author
        voice_channel = author.voice_channel
        channel = ctx.message.channel

        voice_channel = author.voice.voice_channel
        #if voice_channel is None:
        #    await self.bot.say("You're currently not inside a voice channel.")
        #    return
        url = url.strip("<>")

        if not url.startswith("https://"):
            url = url.replace("/","&%47")
            url = "[0x0E74D3C]" + url

        if url not in url and "youtube" in url:
            parsed_url = urllib.parse.urlparse(url)
            query = urllib.parse.parse_qs(parsed_url.query)
            query.pop("list",None)
            parsed_url = parsed_url.replace(query=urllib.parse.urlencore(query,True))
            url = urllib.parse_urlunparse(parsed_url)

        if self.downloaders:
            dl = Downloader(url)
            self.downloaders.append(dl)
            while self.downloaders[0].is_alive():
                if dl == self.downloaders[0]:
                    break
                await asyncio.sleep(5)

        else:
            dl = Downloader(url, download=True)
            self.downloaders.append(dl)
        dl.get_info()
        sng = dl.song
        e = discord.Embed(title=sng.title,color=0xFFFFFF, url=sng.url).set_footer(text=sng.url)

        player = self.garantee_bot(sng, author.voice.voice_channel)

        est_time = sum(x[1].duration for x in self.playlist[player])
        e.set_image(url=sng.thumbnail)
        e.add_field(name="Duration: ", value=display_time(sng.duration))
        e.add_field(name="Views: ", value="**{}**".format(sng.view))
        e.add_field(name="Estimated to play in: ", value=est_time)
        e.add_field(name="Place in queue: ",value=len(self.playlist[player])+1)
        await self.bot.say(embed=e)
        #self.temporary = dl.song

        dl.run()
        dl.done.wait()
        self.playlist[player].append((author.voice.voice_channel, sng))
        self.downloaders.pop(0)

    def is_playing(self,player,server): #.voice
        if not player.is_voice_connected(server):
            return False
        if player.voice_client_in(server) is None:
            return False
        if not hasattr(player.voice_client_in(server),"audio_player"):
            return False
        if player.voice_client_in(sever).audio_player.is_done():
            return False

        return True

    async def _play(self, player, channel, song, **kwargs):
        voice_client = await self._create_ffmpeg_player(player, channel, song, **kwargs)
        return voice_client

    async def queue_manager(self, player):
        channel = self.playlist[player][0][0]
        server = channel.server
        queue = self.playlist[player]
        song = queue[0][1]
        print(self.is_playing(player,server))
        if not self.is_playing(player,server):
            self.skip_votes[player] = []
            voice_client = await self._play(player, channel, song)
            voice_client.audio_player.start()
            self.playlist[player].pop(0)
    async def song_is_finished(self, player, server):
        while not player.voice_client_in(server).audio_player.is_done():
            await asyncio.sleep(0.5)
        return True

    async def queue_scheduler(self):
        while self == self.bot.get_cog('Maudio'):
            tasks = []
            queue = self.playlist
            #temp_queue = copy.deepcopy(self.tempqueue)
            for acc,pl in queue.items():
                if len(pl) == 0:# and len(temp_queue[acc]) == 0
                    continue

                tasks.append(self.bot.loop.create_task(self.queue_manager(acc)))
            completed = [t.done() for t in tasks]
            while not all(completed):
                completed = [t.done() for t in tasks]
                await asyncio.sleep(0.5)
            await asyncio.sleep(1)

    async def reload_monitor(self):
        while self == self.bot.get_cog('Maudio'):
            await asyncio.sleep(1)

        for acc in self.bot_players:
            for vc in acc.voice_clients:
                try:
                    vc.audio_player.stop()
                except:
                    pass

class Downloader(threading.Thread):
    def __init__(self, url, download=False, *args, **kwargs):
        super().__init__(*args,**kwargs)
        self.url = url
        self.done = threading.Event()
        self.song = None
        self._download = download
        self._yt = None
        self.error = None

    def search(self):
        if self._yt is None:
            self._yt = youtube_dl.YoutubeDL(youtube_dl_options)
        if "[0x0E74D3C]" not in self.url:
            video = self._yt.extract_info(self.url, download=False, process=False)
        else:
            self.url = self.url[11:]
            search_list = self._yt.extract_info(self.url, download=False)
            if not "entries" in yt_id.keys():
                video = self._yt.extract_info("https://youtube.com/watch?v={}".format(search_list["id"]), download=False)
                self.song = Song(**video)
                return self.song
            return search_list["entries"]
    def run(self):
        self.get_info()
        if self._download:
            self.download()
        #except youtub_dl.utils.DownloadError as e:
        #    self.error = str(e)
        #except OSError as e:
        #    log.warning("Os error while downloading '{}':\n{}".format(self.url, str(e)))
        self.done.set()

    def download(self):
        if not os.path.isfile("data/audio/shared"+self.song.id):
            video = self._yt.extract_info(self.url)
            self.song = Song(**video)

    def get_info(self):
        if self._yt is None:
            self._yt = youtube_dl.YoutubeDL(youtube_dl_options)
        if "[0x0E74D3C]" not in self.url:
            video = self._yt.extract_info(self.url, download=False, process=False)
        else:
            self.url = self.url[11:]
            yt_id = self._yt.extract_info(self.url, download=False)
            if yt_id.get("entries"):
                yt_id = yt_id["entries"][0]["id"]

            self.url = "https://youtube.com/watch?v={}".format(yt_id)
            video = self._yt.extract_info(yt_id, download=False)

        if video is not None:
            self.song = Song(**video)
class Song:
    def __init__(self, **kwargs):
        self.__dict__ = kwargs
        self.view = kwargs.pop('view_count',None)
        self.description = kwargs.pop('description',None)
        self.likes = kwargs.pop('like_count',None)
        self.thumbnail = kwargs.pop('thumbnail',None)
        self.dislikes = kwargs.pop('dislike_count',None)
        self.view = kwargs.pop('view',None)
        self.title = kwargs.pop('title',None)
        self.id = kwargs.pop('id',None)
        self.url = kwargs.pop('webpage_url',None)
        self.uploader = kwargs.pop('uploader',None)
        self.duration = kwargs.pop('duration',60)
        self.start_time = kwargs.pop('start_time',None)
        self.end_time = kwargs.pop('end_time',None)

def check_folder():
    folders = ("data/audio","data/audio/shared")
    for folder in folders:
        if not os.path.exists(folder):
            print("Creating {} folder.....".format(folder))
            os.makedirs(folder)

def check_files():
    check_folder()
    default = {"VOLUME":50, "AVCONV":False, "VOTE_THRESHOLD":50}
    settings_path = "data/audio/shared/settings.json"

    if not os.path.isfile(settings_path):
        print("Creating default audio settings.json...")
        dataIO.save_json(settings_path, default)
    else:
        try:
            current = dataIO.load_json(settings_path)
        except JSONDecodeError:
            dataIO.save_json(settings_path, default)
            current = dataIO.load_json(settings_path)
        if current.keys() != default.keys():
            for key in default.keys():
                if key not in current.keys():
                    current[key] = default[key]
                    print(
                        "Adding {} field to audio settings.json...".format(key)
                    )
            dataIO.save_json(settings_path, current)

def setup(bot):
    check_files()
    n = Maudio(bot)
    bot.add_cog(n)
    bot.loop.create_task(n.reload_monitor())
    bot.loop.create_task(n.queue_scheduler())