import importlib
import itertools
import random

from sqlalchemy import and_
from sqlalchemy import Boolean
from sqlalchemy import case
from sqlalchemy import cast
from sqlalchemy import Column
from sqlalchemy import column
from sqlalchemy import dialects
from sqlalchemy import exists
from sqlalchemy import extract
from sqlalchemy import Float
from sqlalchemy import Integer
from sqlalchemy import literal
from sqlalchemy import literal_column
from sqlalchemy import MetaData
from sqlalchemy import or_
from sqlalchemy import PickleType
from sqlalchemy import select
from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy import table
from sqlalchemy import testing
from sqlalchemy import text
from sqlalchemy import tuple_
from sqlalchemy import TypeDecorator
from sqlalchemy import union
from sqlalchemy import union_all
from sqlalchemy import values
from sqlalchemy.dialects import mysql
from sqlalchemy.dialects import postgresql
from sqlalchemy.schema import Sequence
from sqlalchemy.sql import bindparam
from sqlalchemy.sql import ColumnElement
from sqlalchemy.sql import dml
from sqlalchemy.sql import False_
from sqlalchemy.sql import func
from sqlalchemy.sql import operators
from sqlalchemy.sql import roles
from sqlalchemy.sql import True_
from sqlalchemy.sql import type_coerce
from sqlalchemy.sql import visitors
from sqlalchemy.sql.base import HasCacheKey
from sqlalchemy.sql.elements import _label_reference
from sqlalchemy.sql.elements import _textual_label_reference
from sqlalchemy.sql.elements import Annotated
from sqlalchemy.sql.elements import BindParameter
from sqlalchemy.sql.elements import ClauseElement
from sqlalchemy.sql.elements import ClauseList
from sqlalchemy.sql.elements import CollationClause
from sqlalchemy.sql.elements import Immutable
from sqlalchemy.sql.elements import Null
from sqlalchemy.sql.elements import Slice
from sqlalchemy.sql.elements import UnaryExpression
from sqlalchemy.sql.functions import FunctionElement
from sqlalchemy.sql.functions import GenericFunction
from sqlalchemy.sql.functions import ReturnTypeFromArgs
from sqlalchemy.sql.lambdas import lambda_stmt
from sqlalchemy.sql.lambdas import LambdaElement
from sqlalchemy.sql.lambdas import LambdaOptions
from sqlalchemy.sql.selectable import _OffsetLimitParam
from sqlalchemy.sql.selectable import AliasedReturnsRows
from sqlalchemy.sql.selectable import FromGrouping
from sqlalchemy.sql.selectable import LABEL_STYLE_NONE
from sqlalchemy.sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL
from sqlalchemy.sql.selectable import Select
from sqlalchemy.sql.selectable import Selectable
from sqlalchemy.sql.selectable import SelectStatementGrouping
from sqlalchemy.sql.type_api import UserDefinedType
from sqlalchemy.sql.visitors import InternalTraversal
from sqlalchemy.testing import eq_
from sqlalchemy.testing import fixtures
from sqlalchemy.testing import is_
from sqlalchemy.testing import is_false
from sqlalchemy.testing import is_not
from sqlalchemy.testing import is_true
from sqlalchemy.testing import ne_
from sqlalchemy.testing.assertions import expect_warnings
from sqlalchemy.testing.util import random_choices
from sqlalchemy.types import ARRAY
from sqlalchemy.types import JSON
from sqlalchemy.util import class_hierarchy

meta = MetaData()
meta2 = MetaData()

table_a = Table("a", meta, Column("a", Integer), Column("b", String))
table_b_like_a = Table("b2", meta, Column("a", Integer), Column("b", String))

table_a_2 = Table("a", meta2, Column("a", Integer), Column("b", String))

table_a_2_fs = Table(
    "a", meta2, Column("a", Integer), Column("b", String), schema="fs"
)
table_a_2_bs = Table(
    "a", meta2, Column("a", Integer), Column("b", String), schema="bs"
)

table_b = Table("b", meta, Column("a", Integer), Column("b", Integer))

table_b_b = Table(
    "b_b",
    meta,
    Column("a", Integer),
    Column("b", Integer),
    Column("c", Integer),
    Column("d", Integer),
    Column("e", Integer),
)

table_c = Table("c", meta, Column("x", Integer), Column("y", Integer))

table_d = Table("d", meta, Column("y", Integer), Column("z", Integer))

def opt1(ctx):
    pass

def opt2(ctx):
    pass

def opt3(ctx):
    pass

class MyEntity(HasCacheKey):
    def __init__(self, name, element):
        self.name = name
        self.element = element

    _cache_key_traversal = [
        ("name", InternalTraversal.dp_string),
        ("element", InternalTraversal.dp_clauseelement),
    ]

class Foo:
    x = 10
    y = 15

dml.Insert.argument_for("sqlite", "foo", None)
dml.Update.argument_for("sqlite", "foo", None)
dml.Delete.argument_for("sqlite", "foo", None)

class MyType1(TypeDecorator):
    cache_ok = True
    impl = String

class MyType2(TypeDecorator):
    cache_ok = True
    impl = Integer

class MyType3(TypeDecorator):
    impl = Integer

    cache_ok = True

    def __init__(self, arg):
        self.arg = arg

class CoreFixtures:
    # lambdas which return a tuple of ColumnElement objects.
    # must return at least two objects that should compare differently.
    # to test more varieties of "difference" additional objects can be added.
    fixtures = [
        lambda: (
            column("q"),
            column("x"),
            column("q", Integer),
            column("q", String),
        ),
        lambda: (~column("q", Boolean), ~column("p", Boolean)),
        lambda: (
            table_a.c.a.label("foo"),
            table_a.c.a.label("bar"),
            table_a.c.b.label("foo"),
        ),
        lambda: (
            _label_reference(table_a.c.a.desc()),
            _label_reference(table_a.c.a.asc()),
        ),
        lambda: (_textual_label_reference("a"), _textual_label_reference("b")),
        lambda: (
            text("select a, b from table").columns(a=Integer, b=String),
            text("select a, b, c from table").columns(
                a=Integer, b=String, c=Integer
            ),
            text("select a, b, c from table where foo=:bar").bindparams(
                bindparam("bar", type_=Integer)
            ),
            text("select a, b, c from table where foo=:foo").bindparams(
                bindparam("foo", type_=Integer)
            ),
            text("select a, b, c from table where foo=:bar").bindparams(
                bindparam("bar", type_=String)
            ),
        ),
        lambda: (
            literal(1).op("+")(literal(1)),
            literal(1).op("-")(literal(1)),
            column("q").op("-")(literal(1)),
            UnaryExpression(table_a.c.b, modifier=operators.neg),
            UnaryExpression(table_a.c.b, modifier=operators.desc_op),
            UnaryExpression(table_a.c.b, modifier=operators.custom_op("!")),
            UnaryExpression(table_a.c.b, modifier=operators.custom_op("~")),
        ),
        lambda: (
            column("q") == column("x"),
            column("q") == column("y"),
            column("z") == column("x"),
            (column("z") == column("x")).self_group(),
            (column("q") == column("x")).self_group(),
            column("z") + column("x"),
            column("z").op("foo")(column("x")),
            column("z").op("foo")(literal(1)),
            column("z").op("bar")(column("x")),
            column("z") - column("x"),
            column("x") - column("z"),
            column("z") > column("x"),
            column("x").in_([5, 7]),
            column("x").in_([10, 7, 8]),
            # note these two are mathematically equivalent but for now they
            # are considered to be different
            column("z") >= column("x"),
            column("x") <= column("z"),
            column("q").between(5, 6),
            column("q").between(5, 6, symmetric=True),
            column("q").like("somstr"),
            column("q").like("somstr", escape="\\"),
            column("q").like("somstr", escape="X"),
        ),
        lambda: (
            column("q").regexp_match("y", flags="ig"),
            column("q").regexp_match("y", flags="q"),
            column("q").regexp_match("y"),
            column("q").regexp_replace("y", "z", flags="ig"),
            column("q").regexp_replace("y", "z", flags="q"),
            column("q").regexp_replace("y", "z"),
        ),
        lambda: (
            column("q", ARRAY(Integer))[3] == 5,
            column("q", ARRAY(Integer))[3:5] == 5,
        ),
        lambda: (
            table_a.c.a,
            table_a.c.a._annotate({"orm": True}),
            table_a.c.a._annotate({"orm": True})._annotate({"bar": False}),
            table_a.c.a._annotate(
                {"orm": True, "parententity": MyEntity("a", table_a)}
            ),
            table_a.c.a._annotate(
                {"orm": True, "parententity": MyEntity("b", table_a)}
            ),
            table_a.c.a._annotate(
                {"orm": True, "parententity": MyEntity("b", select(table_a))}
            ),
            table_a.c.a._annotate(
                {
                    "orm": True,
                    "parententity": MyEntity(
                        "b", select(table_a).where(table_a.c.a == 5)
                    ),
                }
            ),
        ),
        lambda: (
            table_a,
            table_a._annotate({"orm": True}),
            table_a._annotate({"orm": True})._annotate({"bar": False}),
            table_a._annotate(
                {"orm": True, "parententity": MyEntity("a", table_a)}
            ),
            table_a._annotate(
                {"orm": True, "parententity": MyEntity("b", table_a)}
            ),
            table_a._annotate(
                {"orm": True, "parententity": MyEntity("b", select(table_a))}
            ),
        ),
        lambda: (
            table("a", column("x"), column("y")),
            table("a", column("x"), column("y"), schema="q"),
            table("a", column("x"), column("y"), schema="y"),
            table("a", column("x"), column("y"))._annotate({"orm": True}),
            table("b", column("x"), column("y"))._annotate({"orm": True}),
        ),
        lambda: (
            cast(column("q"), Integer),
            cast(column("q"), Float),
            cast(column("p"), Integer),
        ),
        lambda: (
            column("x", JSON)["key1"],
            column("x", JSON)["key1"].as_boolean(),
            column("x", JSON)["key1"].as_float(),
            column("x", JSON)["key1"].as_integer(),
            column("x", JSON)["key1"].as_string(),
            column("y", JSON)["key1"].as_integer(),
            column("y", JSON)["key1"].as_string(),
        ),
        lambda: (
            bindparam("x"),
            bindparam("x", literal_execute=True),
            bindparam("y"),
            bindparam("x", type_=Integer),
            bindparam("x", type_=String),
            bindparam(None),
        ),
        lambda: (_OffsetLimitParam("x"), _OffsetLimitParam("y")),
        lambda: (func.foo(), func.foo(5), func.bar()),
        lambda: (
            func.package1.foo(5),
            func.package2.foo(5),
            func.packge1.bar(5),
            func.foo(),
        ),
        lambda: (func.current_date(), func.current_time()),
        lambda: (
            func.next_value(Sequence("q")),
            func.next_value(Sequence("p")),
        ),
        lambda: (
            func.json_to_recordset("{foo}"),
            func.json_to_recordset("{foo}").table_valued("a", "b"),
            func.jsonb_to_recordset("{foo}").table_valued("a", "b"),
            func.json_to_recordset("{foo}")
            .table_valued("a", "b")
            .render_derived(),
            func.json_to_recordset("{foo}")
            .table_valued("a", with_ordinality="b")
            .render_derived(),
            func.json_to_recordset("{foo}")
            .table_valued("a", with_ordinality="c")
            .render_derived(),
            func.json_to_recordset("{foo}")
            .table_valued(column("a", Integer), column("b", String))
            .render_derived(),
            func.json_to_recordset("{foo}")
            .table_valued(column("a", Integer), column("b", String))
            .render_derived(with_types=True),
            func.json_to_recordset("{foo}")
            .table_valued("b", "c")
            .render_derived(),
            func.json_to_recordset("{foo}")
            .table_valued("a", "b")
            .alias("foo")
            .render_derived(with_types=True),
            func.json_to_recordset("{foo}")
            .table_valued("a", "b")
            .alias("foo"),
            func.json_to_recordset("{foo}").column_valued(),
            func.json_to_recordset("{foo}").scalar_table_valued("foo"),
        ),
        lambda: (table_a.table_valued(), table_b.table_valued()),
        lambda: (True_(), False_()),
        lambda: (Null(),),
        lambda: (ReturnTypeFromArgs("foo"), ReturnTypeFromArgs(5)),
        lambda: (FunctionElement(5), FunctionElement(5, 6)),
        lambda: (func.count(), func.not_count()),
        lambda: (func.char_length("abc"), func.char_length("def")),
        lambda: (GenericFunction("a", "b"), GenericFunction("a")),
        lambda: (CollationClause("foobar"), CollationClause("batbar")),
        lambda: (
            type_coerce(column("q", Integer), String),
            type_coerce(column("q", Integer), Float),
            type_coerce(column("z", Integer), Float),
        ),
        lambda: (table_a.c.a, table_b.c.a),
        lambda: (tuple_(1, 2), tuple_(3, 4)),
        lambda: (func.array_agg([1, 2]), func.array_agg([3, 4])),
        lambda: (
            func.aggregate_strings(table_a.c.b, ","),
            func.aggregate_strings(table_b_like_a.c.b, ","),
        ),
        lambda: (
            func.percentile_cont(0.5).within_group(table_a.c.a),
            func.percentile_cont(0.5).within_group(table_a.c.b),
            func.percentile_cont(0.5).within_group(table_a.c.a, table_a.c.b),
            func.percentile_cont(0.5).within_group(
                table_a.c.a, table_a.c.b, column("q")
            ),
        ),
        lambda: (
            func.is_equal("a", "b").as_comparison(1, 2),
            func.is_equal("a", "c").as_comparison(1, 2),
            func.is_equal("a", "b").as_comparison(2, 1),
            func.is_equal("a", "b", "c").as_comparison(1, 2),
            func.foobar("a", "b").as_comparison(1, 2),
        ),
        lambda: (
            func.row_number().over(order_by=table_a.c.a),
            func.row_number().over(order_by=table_a.c.a, range_=(0, 10)),
            func.row_number().over(order_by=table_a.c.a, range_=(None, 10)),
            func.row_number().over(order_by=table_a.c.a, rows=(None, 20)),
            func.row_number().over(order_by=table_a.c.b),
            func.row_number().over(
                order_by=table_a.c.a, partition_by=table_a.c.b
            ),
        ),
        lambda: (
            func.count(1).filter(table_a.c.a == 5),
            func.count(1).filter(table_a.c.a == 10),
            func.foob(1).filter(table_a.c.a == 10),
        ),
        lambda: (
            and_(table_a.c.a == 5, table_a.c.b == table_b.c.a),
            and_(table_a.c.a == 5, table_a.c.a == table_b.c.a),
            or_(table_a.c.a == 5, table_a.c.b == table_b.c.a),
            ClauseList(table_a.c.a == 5, table_a.c.b == table_b.c.a),
            ClauseList(table_a.c.a == 5, table_a.c.b == table_a.c.a),
        ),
        lambda: (
            case((table_a.c.a == 5, 10), (table_a.c.a == 10, 20)),
            case((table_a.c.a == 18, 10), (table_a.c.a == 10, 20)),
            case((table_a.c.a == 5, 10), (table_a.c.b == 10, 20)),
            case(
                (table_a.c.a == 5, 10),
                (table_a.c.b == 10, 20),
                (table_a.c.a == 9, 12),
            ),
            case(
                (table_a.c.a == 5, 10),
                (table_a.c.a == 10, 20),
                else_=30,
            ),
            case({"wendy": "W", "jack": "J"}, value=table_a.c.a, else_="E"),
            case({"wendy": "W", "jack": "J"}, value=table_a.c.b, else_="E"),
            case({"wendy_w": "W", "jack": "J"}, value=table_a.c.a, else_="E"),
        ),
        lambda: (
            extract("foo", table_a.c.a),
            extract("foo", table_a.c.b),
            extract("bar", table_a.c.a),
        ),
        lambda: (
            Slice(1, 2, 5),
            Slice(1, 5, 5),
            Slice(1, 5, 10),
            Slice(2, 10, 15),
        ),
        lambda: (
            select(table_a.c.a),
            select(table_a.c.a, table_a.c.b),
            select(table_a.c.b, table_a.c.a),
            select(table_a.c.b, table_a.c.a).limit(5),
            select(table_a.c.b, table_a.c.a).limit(5).offset(10),
            select(table_a.c.b, table_a.c.a)
            .limit(literal_column("foobar"))
            .offset(10),
            select(table_a.c.b, table_a.c.a).set_label_style(
                LABEL_STYLE_TABLENAME_PLUS_COL
            ),
            select(table_a.c.b, table_a.c.a).set_label_style(LABEL_STYLE_NONE),
            select(table_a.c.a).where(table_a.c.b == 5),
            select(table_a.c.a)
            .where(table_a.c.b == 5)
            .where(table_a.c.a == 10),
            select(table_a.c.a).where(table_a.c.b == 5).with_for_update(),
            select(table_a.c.a)
            .where(table_a.c.b == 5)
            .with_for_update(nowait=True),
            select(table_a.c.a).where(table_a.c.b == 5).correlate(table_b),
            select(table_a.c.a)
            .where(table_a.c.b == 5)
            .correlate_except(table_b),
        ),
        lambda: (
            select(table_a.c.a),
            select(table_a.c.a).limit(2),
            select(table_a.c.a).limit(3),
            select(table_a.c.a).fetch(3),
            select(table_a.c.a).fetch(2),
            select(table_a.c.a).fetch(2, percent=True),
            select(table_a.c.a).fetch(2, with_ties=True),
            select(table_a.c.a).fetch(2, with_ties=True, percent=True),
            select(table_a.c.a).fetch(2).offset(3),
            select(table_a.c.a).fetch(2).offset(5),
            select(table_a.c.a).limit(2).offset(5),
            select(table_a.c.a).limit(2).offset(3),
            select(table_a.c.a).union(select(table_a.c.a)).limit(2).offset(3),
            union(select(table_a.c.a), select(table_a.c.b)).limit(2).offset(3),
            union(select(table_a.c.a), select(table_a.c.b)).limit(6).offset(3),
            union(select(table_a.c.a), select(table_a.c.b)).limit(6).offset(8),
            union(select(table_a.c.a), select(table_a.c.b)).fetch(2).offset(8),
            union(select(table_a.c.a), select(table_a.c.b)).fetch(6).offset(8),
            union(select(table_a.c.a), select(table_a.c.b)).fetch(6).offset(3),
            union(select(table_a.c.a), select(table_a.c.b))
            .fetch(6, percent=True)
            .offset(3),
            union(select(table_a.c.a), select(table_a.c.b))
            .fetch(6, with_ties=True)
            .offset(3),
            union(select(table_a.c.a), select(table_a.c.b))
            .fetch(6, with_ties=True, percent=True)
            .offset(3),
            union(select(table_a.c.a), select(table_a.c.b)).limit(6),
            union(select(table_a.c.a), select(table_a.c.b)).offset(6),
        ),
        lambda: (
            select(table_a.c.a),
            select(table_a.c.a).join(table_b, table_a.c.a == table_b.c.a),
            select(table_a.c.a).join_from(
                table_a, table_b, table_a.c.a == table_b.c.a
            ),
            select(table_a.c.a).join_from(table_a, table_b),
            select(table_a.c.a).join_from(table_c, table_b),
            select(table_a.c.a)
            .join(table_b, table_a.c.a == table_b.c.a)
            .join(table_c, table_b.c.b == table_c.c.x),
            select(table_a.c.a).join(table_b),
            select(table_a.c.a).join(table_c),
            select(table_a.c.a).join(table_b, table_a.c.a == table_b.c.b),
            select(table_a.c.a).join(table_c, table_a.c.a == table_c.c.x),
        ),
        lambda: (
            select(table_a.c.a),
            select(table_a.c.a).add_cte(table_b.insert().cte()),
            table_a.insert(),
            table_a.delete(),
            table_a.update(),
            table_a.insert().add_cte(table_b.insert().cte()),
            table_a.delete().add_cte(table_b.insert().cte()),
            table_a.update().add_cte(table_b.insert().cte()),
        ),
        lambda: (
            select(table_a.c.a).cte(),
            select(table_a.c.a).cte(nesting=True),
            select(table_a.c.a).cte(recursive=True),
            select(table_a.c.a).cte(name="some_cte", recursive=True),
            select(table_a.c.a).cte(name="some_cte"),
            select(table_a.c.a).cte(name="some_cte").alias("other_cte"),
            select(table_a.c.a)
            .cte(name="some_cte")
            .union_all(select(table_a.c.a)),
            select(table_a.c.a)
            .cte(name="some_cte")
            .union_all(select(table_a.c.b)),
            select(table_a.c.a).lateral(),
            select(table_a.c.a).lateral(name="bar"),
            table_a.tablesample(0.75),
            table_a.tablesample(func.bernoulli(1)),
            table_a.tablesample(func.bernoulli(1), seed=func.random()),
            table_a.tablesample(func.bernoulli(1), seed=func.other_random()),
            table_a.tablesample(func.hoho(1)),
            table_a.tablesample(func.bernoulli(1), name="bar"),
            table_a.tablesample(
                func.bernoulli(1), name="bar", seed=func.random()
            ),
        ),
        lambda: (
            # test issue #6503
            # join from table_a -> table_c, select table_b.c.a
            select(table_a).join(table_c).with_only_columns(table_b.c.a),
            # join from table_b -> table_c, select table_b.c.a
            select(table_b.c.a).join(table_c).with_only_columns(table_b.c.a),
            select(table_a).with_only_columns(table_b.c.a),
        ),
        lambda: (
            table_a.insert(),
            table_a.insert().return_defaults(),
            table_a.insert().return_defaults(table_a.c.a),
            table_a.insert().return_defaults(table_a.c.b),
            table_a.insert().values({})._annotate({"nocache": True}),
            table_b.insert(),
            table_b.insert().with_dialect_options(sqlite_foo="some value"),
            table_b.insert().from_select(["a", "b"], select(table_a)),
            table_b.insert().from_select(
                ["a", "b"], select(table_a).where(table_a.c.a > 5)
            ),
            table_b.insert().from_select(["a", "b"], select(table_b)),
            table_b.insert().from_select(["c", "d"], select(table_a)),
            table_b.insert().returning(table_b.c.a),
            table_b.insert().returning(table_b.c.a, table_b.c.b),
            table_b.insert().inline(),
            table_b.insert().prefix_with("foo"),
            table_b.insert().with_hint("RUNFAST"),
            table_b.insert().values(a=5, b=10),
            table_b.insert().values(a=5),
            table_b.insert()
            .values({table_b.c.a: 5, "b": 10})
            ._annotate({"nocache": True}),
            table_b.insert().values(a=7, b=10),
            table_b.insert().values(a=5, b=10).inline(),
            table_b.insert()
            .values([{"a": 5, "b": 10}, {"a": 8, "b": 12}])
            ._annotate({"nocache": True}),
            table_b.insert()
            .values([{"a": 9, "b": 10}, {"a": 8, "b": 7}])
            ._annotate({"nocache": True}),
            table_b.insert()
            .values([(5, 10), (8, 12)])
            ._annotate({"nocache": True}),
            table_b.insert()
            .values([(5, 9), (5, 12)])
            ._annotate({"nocache": True}),
        ),
        lambda: (
            table_b.update(),
            table_b.update().return_defaults(),
            table_b.update().return_defaults(table_b.c.a),
            table_b.update().return_defaults(table_b.c.b),
            table_b.update().where(table_b.c.a == 5),
            table_b.update().where(table_b.c.b == 5),
            table_b.update()
            .where(table_b.c.b == 5)
            .with_dialect_options(mysql_limit=10),
            table_b.update()
            .where(table_b.c.b == 5)
            .with_dialect_options(mysql_limit=10, sqlite_foo="some value"),
            table_b.update().where(table_b.c.a == 5).values(a=5, b=10),
            table_b.update().where(table_b.c.a == 5).values(a=5, b=10, c=12),
            table_b.update()
            .where(table_b.c.b == 5)
            .values(a=5, b=10)
            ._annotate({"nocache": True}),
            table_b.update().values(a=5, b=10),
            table_b.update()
            .values({"a": 5, table_b.c.b: 10})
            ._annotate({"nocache": True}),
            table_b.update().values(a=7, b=10),
            table_b.update().ordered_values(("a", 5), ("b", 10)),
            table_b.update().ordered_values(("b", 10), ("a", 5)),
            table_b.update().ordered_values((table_b.c.a, 5), ("b", 10)),
        ),
        lambda: (
            table_b.delete(),
            table_b.delete().with_dialect_options(sqlite_foo="some value"),
            table_b.delete().where(table_b.c.a == 5),
            table_b.delete().where(table_b.c.b == 5),
        ),
        lambda: (
            values(
                column("mykey", Integer),
                column("mytext", String),
                column("myint", Integer),
                name="myvalues",
            )
            .data([(1, "textA", 99), (2, "textB", 88)])
            ._annotate({"nocache": True}),
            values(
                column("mykey", Integer),
                column("mytext", String),
                column("myint", Integer),
                name="myvalues",
                literal_binds=True,
            )
            .data([(1, "textA", 99), (2, "textB", 88)])
            ._annotate({"nocache": True}),
            values(
                column("mykey", Integer),
                column("mytext", String),
                column("myint", Integer),
                name="myothervalues",
            )
            .data([(1, "textA", 99), (2, "textB", 88)])
            ._annotate({"nocache": True}),
            values(
                column("mykey", Integer),
                column("mytext", String),
                column("myint", Integer),
                name="myvalues",
            )
            .data([(1, "textA", 89), (2, "textG", 88)])
            ._annotate({"nocache": True}),
            values(
                column("mykey", Integer),
                column("mynottext", String),
                column("myint", Integer),
                name="myvalues",
            )
            .data([(1, "textA", 99), (2, "textB", 88)])
            ._annotate({"nocache": True}),
            # TODO: difference in type
            # values(
            #     column("mykey", Integer),
            #     column("mytext", Text),
            #     column("myint", Integer),
            #     name="myvalues",
            # )
            # .data([(1, "textA", 99), (2, "textB", 88)])
            # ._annotate({"nocache": True}),
        ),
        lambda: (
            values(
                column("mykey", Integer),
                column("mytext", String),
                column("myint", Integer),
                name="myvalues",
            )
            .data([(1, "textA", 99), (2, "textB", 88)])
            .scalar_values()
            ._annotate({"nocache": True}),
            values(
                column("mykey", Integer),
                column("mytext", String),
                column("myint", Integer),
                name="myvalues",
                literal_binds=True,
            )
            .data([(1, "textA", 99), (2, "textB", 88)])
            .scalar_values()
            ._annotate({"nocache": True}),
            values(
                column("mykey", Integer),
                column("mytext", String),
                column("myint", Integer),
                name="myvalues",
            )
            .data([(1, "textA", 89), (2, "textG", 88)])
            .scalar_values()
            ._annotate({"nocache": True}),
            values(
                column("mykey", Integer),
                column("mynottext", String),
                column("myint", Integer),
                name="myvalues",
            )
            .data([(1, "textA", 99), (2, "textB", 88)])
            .scalar_values()
            ._annotate({"nocache": True}),
            # TODO: difference in type
            # values(
            #     column("mykey", Integer),
            #     column("mytext", Text),
            #     column("myint", Integer),
            #     name="myvalues",
            # )
            # .data([(1, "textA", 99), (2, "textB", 88)])
            # .scalar_values()
            # ._annotate({"nocache": True}),
        ),
        lambda: (
            select(table_a.c.a),
            select(table_a.c.a).prefix_with("foo"),
            select(table_a.c.a).prefix_with("foo", dialect="mysql"),
            select(table_a.c.a).prefix_with("foo", dialect="postgresql"),
            select(table_a.c.a).prefix_with("bar"),
            select(table_a.c.a).suffix_with("bar"),
        ),
        lambda: (
            select(table_a_2.c.a),
            select(table_a_2_fs.c.a),
            select(table_a_2_bs.c.a),
        ),
        lambda: (
            select(table_a.c.a),
            select(table_a.c.a).with_hint(None, "some hint"),
            select(table_a.c.a).with_hint(None, "some other hint"),
            select(table_a.c.a).with_hint(table_a, "some hint"),
            select(table_a.c.a)
            .with_hint(table_a, "some hint")
            .with_hint(None, "some other hint"),
            select(table_a.c.a).with_hint(table_a, "some other hint"),
            select(table_a.c.a).with_hint(
                table_a, "some hint", dialect_name="mysql"
            ),
            select(table_a.c.a).with_hint(
                table_a, "some hint", dialect_name="postgresql"
            ),
        ),
        lambda: (
            table_a.join(table_b, table_a.c.a == table_b.c.a),
            table_a.join(
                table_b, and_(table_a.c.a == table_b.c.a, table_a.c.b == 1)
            ),
            table_a.outerjoin(table_b, table_a.c.a == table_b.c.a),
        ),
        lambda: (
            table_a.alias("a"),
            table_a.alias("b"),
            table_a.alias(),
            table_b.alias("a"),
            select(table_a.c.a).alias("a"),
        ),
        lambda: (
            FromGrouping(table_a.alias("a")),
            FromGrouping(table_a.alias("b")),
        ),
        lambda: (
            SelectStatementGrouping(select(table_a)),
            SelectStatementGrouping(select(table_b)),
        ),
        lambda: (
            select(table_a.c.a).scalar_subquery(),
            select(table_a.c.a).where(table_a.c.b == 5).scalar_subquery(),
        ),
        lambda: (
            exists().where(table_a.c.a == 5),
            exists().where(table_a.c.b == 5),
        ),
        lambda: (
            union(select(table_a.c.a), select(table_a.c.b)),
            union(select(table_a.c.a), select(table_a.c.b)).order_by("a"),
            union_all(select(table_a.c.a), select(table_a.c.b)),
            union(select(table_a.c.a)),
            union(
                select(table_a.c.a),
                select(table_a.c.b).where(table_a.c.b > 5),
            ),
        ),
        lambda: (
            table("a", column("x"), column("y")),
            table("a", column("y"), column("x")),
            table("b", column("x"), column("y")),
            table("a", column("x"), column("y"), column("z")),
            table("a", column("x"), column("y", Integer)),
            table("a", column("q"), column("y", Integer)),
        ),
        lambda: (table_a, table_b),
    ]

    type_cache_key_fixtures = [
        lambda: (
            column("q") == column("x"),
            column("q") == column("y"),
            column("z") == column("x"),
            column("z", String(50)) == column("x", String(50)),
            column("z", String(50)) == column("x", String(30)),
            column("z", String(50)) == column("x", Integer),
            column("z", MyType1()) == column("x", MyType2()),
            column("z", MyType1()) == column("x", MyType3("x")),
            column("z", MyType1()) == column("x", MyType3("y")),
        )
    ]

    dont_compare_values_fixtures = [
        lambda: (
            # note the in_(...) all have different column names because
            # otherwise all IN expressions would compare as equivalent
            column("x").in_(random_choices(range(10), k=3)),
            column("y").in_(
                bindparam(
                    "q",
                    # note that a different cache key is created if
                    # the value given to the bindparam is [], as the type
                    # cannot be inferred for the empty list but can
                    # for the non-empty list as of #6222
                    random_choices(range(10), k=random.randint(1, 7)),
                    expanding=True,
                )
            ),
            column("y2").in_(
                bindparam(
                    "q",
                    # for typed param, empty and not empty param will have
                    # the same type
                    random_choices(range(10), k=random.randint(0, 7)),
                    type_=Integer,
                    expanding=True,
                )
            ),
            # don't include empty for untyped, will create different cache
            # key
            column("z").in_(random_choices(range(10), k=random.randint(1, 7))),
            # empty is fine for typed, will create the same cache key
            column("z2", Integer).in_(
                random_choices(range(10), k=random.randint(0, 7))
            ),
            column("x") == random.randint(0, 10),
        )
    ]

    def _complex_fixtures():
        def one():
            a1 = table_a.alias()
            a2 = table_b_like_a.alias()

            stmt = (
                select(table_a.c.a, a1.c.b, a2.c.b)
                .where(table_a.c.b == a1.c.b)
                .where(a1.c.b == a2.c.b)
                .where(a1.c.a == 5)
            )

            return stmt

        def one_diff():
            a1 = table_b_like_a.alias()
            a2 = table_a.alias()

            stmt = (
                select(table_a.c.a, a1.c.b, a2.c.b)
                .where(table_a.c.b == a1.c.b)
                .where(a1.c.b == a2.c.b)
                .where(a1.c.a == 5)
            )

            return stmt

        def two():
            inner = one().subquery()

            stmt = select(table_b.c.a, inner.c.a, inner.c.b).select_from(
                table_b.join(inner, table_b.c.b == inner.c.b)
            )

            return stmt

        def three():
            a1 = table_a.alias()
            a2 = table_a.alias()
            ex = exists().where(table_b.c.b == a1.c.a)

            stmt = (
                select(a1.c.a, a2.c.a)
                .select_from(a1.join(a2, a1.c.b == a2.c.b))
                .where(ex)
            )
            return stmt

        def four():
            stmt = select(table_a.c.a).cte(recursive=True)
            stmt = stmt.union(select(stmt.c.a + 1).where(stmt.c.a < 10))
            return stmt

        def five():
            stmt = select(table_a.c.a).cte(recursive=True, nesting=True)
            stmt = stmt.union(select(stmt.c.a + 1).where(stmt.c.a < 10))
            return stmt

        return [one(), one_diff(), two(), three(), four(), five()]

    fixtures.append(_complex_fixtures)

    def _statements_w_context_options_fixtures():
        return [
            select(table_a)._add_context_option(opt1, True),
            select(table_a)._add_context_option(opt1, 5),
            select(table_a)
            ._add_context_option(opt1, True)
            ._add_context_option(opt2, True),
            select(table_a)
            ._add_context_option(opt1, True)
            ._add_context_option(opt2, 5),
            select(table_a)._add_context_option(opt3, True),
        ]

    fixtures.append(_statements_w_context_options_fixtures)

    def _statements_w_anonymous_col_names():
        def one():
            c = column("q")

            l = c.label(None)

            # new case as of Id810f485c5f7ed971529489b84694e02a3356d6d
            subq = select(l).subquery()

            # this creates a ColumnClause as a proxy to the Label() that has
            # an anonymous name, so the column has one too.
            anon_col = subq.c[0]

            # then when BindParameter is created, it checks the label
            # and doesn't double up on the anonymous name which is uncachable
            return anon_col > 5

        def two():
            c = column("p")

            l = c.label(None)

            # new case as of Id810f485c5f7ed971529489b84694e02a3356d6d
            subq = select(l).subquery()

            # this creates a ColumnClause as a proxy to the Label() that has
            # an anonymous name, so the column has one too.
            anon_col = subq.c[0]

            # then when BindParameter is created, it checks the label
            # and doesn't double up on the anonymous name which is uncachable
            return anon_col > 5

        def three():
            l1, l2 = table_a.c.a.label(None), table_a.c.b.label(None)

            stmt = select(table_a.c.a, table_a.c.b, l1, l2)

            subq = stmt.subquery()
            return select(subq).where(subq.c[2] == 10)

        return (
            one(),
            two(),
            three(),
        )

    fixtures.append(_statements_w_anonymous_col_names)

    def _update_dml_w_dicts():
        return (
            table_b_b.update().values(
                {
                    table_b_b.c.a: 5,
                    table_b_b.c.b: 5,
                    table_b_b.c.c: 5,
                    table_b_b.c.d: 5,
                }
            ),
            # equivalent, but testing dictionary insert ordering as cache key
            # / compare
            table_b_b.update().values(
                {
                    table_b_b.c.a: 5,
                    table_b_b.c.c: 5,
                    table_b_b.c.b: 5,
                    table_b_b.c.d: 5,
                }
            ),
            table_b_b.update().values(
                {table_b_b.c.a: 5, table_b_b.c.b: 5, "c": 5, table_b_b.c.d: 5}
            ),
            table_b_b.update().values(
                {
                    table_b_b.c.a: 5,
                    table_b_b.c.b: 5,
                    table_b_b.c.c: 5,
                    table_b_b.c.d: 5,
                    table_b_b.c.e: 10,
                }
            ),
            table_b_b.update()
            .values(
                {
                    table_b_b.c.a: 5,
                    table_b_b.c.b: 5,
                    table_b_b.c.c: 5,
                    table_b_b.c.d: 5,
                    table_b_b.c.e: 10,
                }
            )
            .where(table_b_b.c.c > 10),
        )

    fixtures.append(_update_dml_w_dicts)

    def _lambda_fixtures():
        def one():
            return LambdaElement(
                lambda: table_a.c.a == column("q"), roles.WhereHavingRole
            )

        def two():
            r = random.randint(1, 10)
            q = 408
            return LambdaElement(
                lambda: table_a.c.a + q == r, roles.WhereHavingRole
            )

        some_value = random.randint(20, 30)

        def three(y):
            return LambdaElement(
                lambda: and_(table_a.c.a == some_value, table_a.c.b > y),
                roles.WhereHavingRole,
            )

        def four():
            return LambdaElement(
                lambda: and_(table_a.c.a == Foo.x), roles.WhereHavingRole
            )

        def five():
            return LambdaElement(
                lambda: and_(table_a.c.a == Foo.x, table_a.c.b == Foo.y),
                roles.WhereHavingRole,
            )

        def six():
            d = {"g": random.randint(40, 45)}

            return LambdaElement(
                lambda: and_(table_a.c.b == d["g"]),
                roles.WhereHavingRole,
                opts=LambdaOptions(track_closure_variables=False),
            )

        def seven():
            # lambda statements don't collect bindparameter objects
            # for fixed values, has to be in a variable
            value = random.randint(10, 20)
            return lambda_stmt(lambda: select(table_a)) + (
                lambda s: s.where(table_a.c.a == value)
            )

        from sqlalchemy.sql import lambdas

        def eight():
            q = 5
            return lambdas.DeferredLambdaElement(
                lambda t: t.c.a > q,
                roles.WhereHavingRole,
                lambda_args=(table_a,),
            )

        return [
            one(),
            two(),
            three(random.randint(5, 10)),
            four(),
            five(),
            six(),
            seven(),
            eight(),
        ]

    dont_compare_values_fixtures.append(_lambda_fixtures)

    # like fixture but returns at least two objects that compare equally
    equal_fixtures = [
        lambda: (
            select(table_a.c.a).fetch(3),
            select(table_a.c.a).fetch(2).fetch(3),
            select(table_a.c.a).fetch(3, percent=False, with_ties=False),
            select(table_a.c.a).limit(2).fetch(3),
            select(table_a.c.a).slice(2, 4).fetch(3).offset(None),
        ),
        lambda: (
            select(table_a.c.a).limit(3),
            select(table_a.c.a).fetch(2).limit(3),
            select(table_a.c.a).fetch(2).slice(0, 3).offset(None),
        ),
    ]

class CacheKeyTest(fixtures.CacheKeyFixture, CoreFixtures, fixtures.TestBase):
    # we are slightly breaking the policy of not having external dialect
    # stuff in here, but use pg/mysql as test cases to ensure that these
    # objects don't report an inaccurate cache key, which is dependent
    # on the base insert sending out _post_values_clause and the caching
    # system properly recognizing these constructs as not cacheable

    @testing.combinations(
        postgresql.insert(table_a).on_conflict_do_update(
            index_elements=[table_a.c.a], set_={"name": "foo"}
        ),
        mysql.insert(table_a).on_duplicate_key_update(updated_once=None),
        table_a.insert().values(  # multivalues doesn't cache
            [
                {"name": "some name"},
                {"name": "some other name"},
                {"name": "yet another name"},
            ]
        ),
    )
    def test_dml_not_cached_yet(self, dml_stmt):
        eq_(dml_stmt._generate_cache_key(), None)

    def test_values_doesnt_caches_right_now(self):
        v1 = values(
            column("mykey", Integer),
            column("mytext", String),
            column("myint", Integer),
            name="myvalues",
        ).data([(1, "textA", 99), (2, "textB", 88)])

        is_(v1._generate_cache_key(), None)

        large_v1 = values(
            column("mykey", Integer),
            column("mytext", String),
            column("myint", Integer),
            name="myvalues",
        ).data([(i, "data %s" % i, i * 5) for i in range(500)])

        is_(large_v1._generate_cache_key(), None)

    @testing.combinations(
        (lambda: column("x"), lambda: column("x"), lambda: column("y")),
        (
            lambda: func.foo_bar(1, 2, 3),
            lambda: func.foo_bar(4, 5, 6),
            lambda: func.foo_bar_bat(1, 2, 3),
        ),
    )
    def test_cache_key_object_comparators(self, lc1, lc2, lc3):
        """test ne issue detected as part of #10042"""
        c1 = lc1()
        c2 = lc2()
        c3 = lc3()

        eq_(c1._generate_cache_key(), c2._generate_cache_key())
        ne_(c1._generate_cache_key(), c3._generate_cache_key())
        is_true(c1._generate_cache_key() == c2._generate_cache_key())
        is_false(c1._generate_cache_key() != c2._generate_cache_key())
        is_true(c1._generate_cache_key() != c3._generate_cache_key())
        is_false(c1._generate_cache_key() == c3._generate_cache_key())

    def test_cache_key(self):
        for fixtures_, compare_values in [
            (self.fixtures, True),
            (self.dont_compare_values_fixtures, False),
            (self.type_cache_key_fixtures, False),
        ]:
            for fixture in fixtures_:
                self._run_cache_key_fixture(fixture, compare_values)

    def test_cache_key_equal(self):
        for fixture in self.equal_fixtures:
            self._run_cache_key_equal_fixture(fixture, True)

    def test_literal_binds(self):
        def fixture():
            return (
                bindparam(None, value="x", literal_execute=True),
                bindparam(None, value="y", literal_execute=True),
            )

        self._run_cache_key_fixture(
            fixture,
            True,
        )

    def test_bindparam_subclass_nocache(self):
        # does not implement inherit_cache
        class _literal_bindparam(BindParameter):
            pass

        l1 = _literal_bindparam(None, value="x1")
        is_(l1._generate_cache_key(), None)

    def test_bindparam_subclass_ok_cache(self):
        # implements inherit_cache
        class _literal_bindparam(BindParameter):
            inherit_cache = True

        def fixture():
            return (
                _literal_bindparam(None, value="x1"),
                _literal_bindparam(None, value="x2"),
                _literal_bindparam(None),
            )

        self._run_cache_key_fixture(fixture, True)

    def test_cache_key_unknown_traverse(self):
        class Foobar1(ClauseElement):
            _traverse_internals = [
                ("key", InternalTraversal.dp_anon_name),
                ("type_", InternalTraversal.dp_unknown_structure),
            ]

            def __init__(self, key, type_):
                self.key = key
                self.type_ = type_

        f1 = Foobar1("foo", String())
        eq_(f1._generate_cache_key(), None)

    def test_cache_key_no_method(self):
        class Foobar1(ClauseElement):
            pass

        class Foobar2(ColumnElement):
            pass

        # the None for cache key will prevent objects
        # which contain these elements from being cached.
        f1 = Foobar1()
        with expect_warnings(
            "Class Foobar1 will not make use of SQL compilation caching"
        ):
            eq_(f1._generate_cache_key(), None)

        f2 = Foobar2()
        with expect_warnings(
            "Class Foobar2 will not make use of SQL compilation caching"
        ):
            eq_(f2._generate_cache_key(), None)

        s1 = select(column("q"), Foobar2())

        # warning is memoized, won't happen the second time
        eq_(s1._generate_cache_key(), None)

    def test_get_children_no_method(self):
        class Foobar1(ClauseElement):
            pass

        class Foobar2(ColumnElement):
            pass

        f1 = Foobar1()
        eq_(f1.get_children(), [])

        f2 = Foobar2()
        eq_(f2.get_children(), [])

    def test_copy_internals_no_method(self):
        class Foobar1(ClauseElement):
            pass

        class Foobar2(ColumnElement):
            pass

        f1 = Foobar1()
        f2 = Foobar2()

        f1._copy_internals()
        f2._copy_internals()

    def test_generative_cache_key_regen(self):
        t1 = table("t1", column("a"), column("b"))

        s1 = select(t1)

        ck1 = s1._generate_cache_key()

        s2 = s1.where(t1.c.a == 5)

        ck2 = s2._generate_cache_key()

        ne_(ck1, ck2)
        is_not(ck1, None)
        is_not(ck2, None)

    def test_generative_cache_key_regen_w_del(self):
        t1 = table("t1", column("a"), column("b"))

        s1 = select(t1)

        ck1 = s1._generate_cache_key()

        s2 = s1.where(t1.c.a == 5)

        del s1

        # there is now a good chance that id(s3) == id(s1), make sure
        # cache key is regenerated

        s3 = s2.order_by(t1.c.b)

        ck3 = s3._generate_cache_key()

        ne_(ck1, ck3)
        is_not(ck1, None)
        is_not(ck3, None)

class CompareAndCopyTest(CoreFixtures, fixtures.TestBase):
    @classmethod
    def setup_test_class(cls):
        # TODO: we need to get dialects here somehow, perhaps in test_suite?
        [
            importlib.import_module("sqlalchemy.dialects.%s" % d)
            for d in dialects.__all__
            if not d.startswith("_")
        ]

    def test_all_present(self):
        """test for elements that are in SQLAlchemy Core, that they are
        also included in the fixtures above.

        """
        need = {
            cls
            for cls in class_hierarchy(ClauseElement)
            if issubclass(cls, (ColumnElement, Selectable, LambdaElement))
            and (
                "__init__" in cls.__dict__
                or issubclass(cls, AliasedReturnsRows)
            )
            and not issubclass(cls, (Annotated))
            and cls.__module__.startswith("sqlalchemy.")
            and "orm" not in cls.__module__
            and "compiler" not in cls.__module__
            and "crud" not in cls.__module__
            and "dialects" not in cls.__module__  # TODO: dialects?
        }.difference({ColumnElement, UnaryExpression})

        for fixture in self.fixtures + self.dont_compare_values_fixtures:
            case_a = fixture()
            for elem in case_a:
                for mro in type(elem).__mro__:
                    need.discard(mro)

        is_false(bool(need), "%d Remaining classes: %r" % (len(need), need))

    def test_compare_labels(self):
        for fixtures_, compare_values in [
            (self.fixtures, True),
            (self.dont_compare_values_fixtures, False),
        ]:
            for fixture in fixtures_:
                case_a = fixture()
                case_b = fixture()

                for a, b in itertools.combinations_with_replacement(
                    range(len(case_a)), 2
                ):
                    if a == b:
                        is_true(
                            case_a[a].compare(
                                case_b[b],
                                compare_annotations=True,
                                compare_values=compare_values,
                            ),
                            f"{case_a[a]!r} != {case_b[b]!r} (index {a} {b})",
                        )
                    else:
                        is_false(
                            case_a[a].compare(
                                case_b[b],
                                compare_annotations=True,
                                compare_values=compare_values,
                            ),
                            f"{case_a[a]!r} == {case_b[b]!r} (index {a} {b})",
                        )

    def test_compare_col_identity(self):
        stmt1 = (
            select(table_a.c.a, table_b.c.b)
            .where(table_a.c.a == table_b.c.b)
            .alias()
        )
        stmt1_c = (
            select(table_a.c.a, table_b.c.b)
            .where(table_a.c.a == table_b.c.b)
            .alias()
        )

        stmt2 = union(select(table_a), select(table_b))

        equivalents = {table_a.c.a: [table_b.c.a]}

        is_false(
            stmt1.compare(stmt2, use_proxies=True, equivalents=equivalents)
        )

        is_true(
            stmt1.compare(stmt1_c, use_proxies=True, equivalents=equivalents)
        )
        is_true(
            (table_a.c.a == table_b.c.b).compare(
                stmt1.c.a == stmt1.c.b,
                use_proxies=True,
                equivalents=equivalents,
            )
        )

    def test_copy_internals(self):
        for fixtures_, compare_values in [
            (self.fixtures, True),
            (self.dont_compare_values_fixtures, False),
        ]:
            for fixture in fixtures_:
                case_a = fixture()
                case_b = fixture()

                for idx in range(len(case_a)):
                    assert case_a[idx].compare(
                        case_b[idx], compare_values=compare_values
                    )

                    clone = visitors.replacement_traverse(
                        case_a[idx], {}, lambda elem: None
                    )

                    assert clone.compare(
                        case_b[idx], compare_values=compare_values
                    )

                    assert case_a[idx].compare(
                        case_b[idx], compare_values=compare_values
                    )

                    # copy internals of Select is very different than other
                    # elements and additionally this is extremely well tested
                    # in test_selectable and test_external_traversal, so
                    # skip these
                    if isinstance(case_a[idx], Select):
                        continue

                    for elema, elemb in zip(
                        visitors.iterate(case_a[idx], {}),
                        visitors.iterate(clone, {}),
                    ):
                        if isinstance(elema, ClauseElement) and not isinstance(
                            elema, Immutable
                        ):
                            assert elema is not elemb

class CompareClausesTest(fixtures.TestBase):
    def test_compare_metadata_tables_annotations_one(self):
        # test that cache keys from annotated version of tables refresh
        # properly

        t1 = Table("a", MetaData(), Column("q", Integer), Column("p", Integer))
        t2 = Table("a", MetaData(), Column("q", Integer), Column("p", Integer))

        ne_(t1._generate_cache_key(), t2._generate_cache_key())

        eq_(t1._generate_cache_key().key, (t1,))

        t2 = t1._annotate({"foo": "bar"})
        eq_(
            t2._generate_cache_key().key,
            (t1, "_annotations", (("foo", "bar"),)),
        )
        eq_(
            t2._annotate({"bat": "bar"})._generate_cache_key().key,
            (t1, "_annotations", (("bat", "bar"), ("foo", "bar"))),
        )

    def test_compare_metadata_tables_annotations_two(self):
        t1 = Table("a", MetaData(), Column("q", Integer), Column("p", Integer))
        t2 = Table("a", MetaData(), Column("q", Integer), Column("p", Integer))

        eq_(t2._generate_cache_key().key, (t2,))

        t1 = t1._annotate({"orm": True})
        t2 = t2._annotate({"orm": True})

        ne_(t1._generate_cache_key(), t2._generate_cache_key())

        eq_(
            t1._generate_cache_key().key,
            (t1, "_annotations", (("orm", True),)),
        )

    def test_compare_adhoc_tables(self):
        # non-metadata tables compare on their structure.  these objects are
        # not commonly used.

        # note this test is a bit redundant as we have a similar test
        # via the fixtures also
        t1 = table("a", Column("q", Integer), Column("p", Integer))
        t2 = table("a", Column("q", Integer), Column("p", Integer))
        t3 = table("b", Column("q", Integer), Column("p", Integer))
        t4 = table("a", Column("q", Integer), Column("x", Integer))

        eq_(t1._generate_cache_key(), t2._generate_cache_key())

        ne_(t1._generate_cache_key(), t3._generate_cache_key())
        ne_(t1._generate_cache_key(), t4._generate_cache_key())
        ne_(t3._generate_cache_key(), t4._generate_cache_key())

    def test_compare_comparison_associative(self):
        l1 = table_c.c.x == table_d.c.y
        l2 = table_d.c.y == table_c.c.x
        l3 = table_c.c.x == table_d.c.z

        is_true(l1.compare(l1))
        is_true(l1.compare(l2))
        is_false(l1.compare(l3))

    def test_compare_comparison_non_commutative_inverses(self):
        l1 = table_c.c.x >= table_d.c.y
        l2 = table_d.c.y < table_c.c.x
        l3 = table_d.c.y <= table_c.c.x

        # we're not doing this kind of commutativity right now.
        is_false(l1.compare(l2))
        is_false(l1.compare(l3))

    def test_compare_clauselist_associative(self):
        l1 = and_(table_c.c.x == table_d.c.y, table_c.c.y == table_d.c.z)

        l2 = and_(table_c.c.y == table_d.c.z, table_c.c.x == table_d.c.y)

        l3 = and_(table_c.c.x == table_d.c.z, table_c.c.y == table_d.c.y)

        is_true(l1.compare(l1))
        is_true(l1.compare(l2))
        is_false(l1.compare(l3))

    def test_compare_clauselist_not_associative(self):
        l1 = ClauseList(
            table_c.c.x, table_c.c.y, table_d.c.y, operator=operators.sub
        )

        l2 = ClauseList(
            table_d.c.y, table_c.c.x, table_c.c.y, operator=operators.sub
        )

        is_true(l1.compare(l1))
        is_false(l1.compare(l2))

    def test_compare_clauselist_assoc_different_operator(self):
        l1 = and_(table_c.c.x == table_d.c.y, table_c.c.y == table_d.c.z)

        l2 = or_(table_c.c.y == table_d.c.z, table_c.c.x == table_d.c.y)

        is_false(l1.compare(l2))

    def test_compare_clauselist_not_assoc_different_operator(self):
        l1 = ClauseList(
            table_c.c.x, table_c.c.y, table_d.c.y, operator=operators.sub
        )

        l2 = ClauseList(
            table_c.c.x, table_c.c.y, table_d.c.y, operator=operators.truediv
        )

        is_false(l1.compare(l2))

    def test_cache_key_limit_offset_values(self):
        s1 = select(column("q")).limit(10)
        s2 = select(column("q")).limit(25)
        s3 = select(column("q")).limit(25).offset(5)
        s4 = select(column("q")).limit(25).offset(18)
        s5 = select(column("q")).limit(7).offset(12)
        s6 = select(column("q")).limit(literal_column("q")).offset(12)

        for should_eq_left, should_eq_right in [(s1, s2), (s3, s4), (s3, s5)]:
            eq_(
                should_eq_left._generate_cache_key().key,
                should_eq_right._generate_cache_key().key,
            )

        for shouldnt_eq_left, shouldnt_eq_right in [
            (s1, s3),
            (s5, s6),
            (s2, s3),
        ]:
            ne_(
                shouldnt_eq_left._generate_cache_key().key,
                shouldnt_eq_right._generate_cache_key().key,
            )

    def test_compare_labels(self):
        is_true(column("q").label(None).compare(column("q").label(None)))

        is_false(column("q").label("foo").compare(column("q").label(None)))

        is_false(column("q").label(None).compare(column("q").label("foo")))

        is_false(column("q").label("foo").compare(column("q").label("bar")))

        is_true(column("q").label("foo").compare(column("q").label("foo")))

    def test_compare_binds(self):
        b1 = bindparam("foo", type_=Integer())
        b1l = bindparam("foo", type_=Integer(), literal_execute=True)
        b2 = bindparam("foo", type_=Integer())
        b3 = bindparam("foo", type_=String())

        def c1():
            return 5

        def c2():
            return 6

        b4 = bindparam("foo", type_=Integer(), callable_=c1)
        b4l = bindparam(
            "foo", type_=Integer(), callable_=c1, literal_execute=True
        )
        b5 = bindparam("foo", type_=Integer(), callable_=c2)
        b6 = bindparam("foo", type_=Integer(), callable_=c1)

        b7 = bindparam("foo", type_=Integer, value=5)
        b8 = bindparam("foo", type_=Integer, value=6)

        is_false(b1.compare(b4))
        is_true(b4.compare(b6))
        is_false(b4.compare(b5))
        is_true(b1.compare(b2))

        # currently not comparing "key", as we often have to compare
        # anonymous names.  however we should really check for that
        # is_true(b1.compare(b3))

        is_false(b1.compare(b3))
        is_false(b1.compare(b7))
        is_false(b7.compare(b8))
        is_true(b7.compare(b7))

        # cache key
        def compare_key(left, right, expected):
            lk = left._generate_cache_key().key
            rk = right._generate_cache_key().key
            is_(lk == rk, expected)

        compare_key(b1, b4, True)
        compare_key(b1, b5, True)
        compare_key(b8, b5, True)
        compare_key(b8, b7, True)
        compare_key(b8, b3, False)
        compare_key(b1, b1l, False)
        compare_key(b1, b4l, False)
        compare_key(b4, b4l, False)
        compare_key(b7, b4l, False)

    def test_compare_tables(self):
        is_true(table_a.compare(table_a_2))

        # the "proxy" version compares schema tables on metadata identity
        is_false(table_a.compare(table_a_2, use_proxies=True))

        # same for lower case tables since it compares lower case columns
        # using proxies, which makes it very unlikely to have multiple
        # table() objects with columns that compare equally
        is_false(
            table("a", column("x", Integer), column("q", String)).compare(
                table("a", column("x", Integer), column("q", String)),
                use_proxies=True,
            )
        )

    def test_compare_annotated_clears_mapping(self):
        t = table("t", column("x"), column("y"))
        x_a = t.c.x._annotate({"foo": True})
        x_b = t.c.x._annotate({"foo": True})

        is_true(x_a.compare(x_b, compare_annotations=True))
        is_false(
            x_a.compare(x_b._annotate({"bar": True}), compare_annotations=True)
        )

        s1 = select(t.c.x)._annotate({"foo": True})
        s2 = select(t.c.x)._annotate({"foo": True})

        is_true(s1.compare(s2, compare_annotations=True))

        is_false(
            s1.compare(s2._annotate({"bar": True}), compare_annotations=True)
        )

    def test_compare_annotated_wo_annotations(self):
        t = table("t", column("x"), column("y"))
        x_a = t.c.x._annotate({})
        x_b = t.c.x._annotate({"foo": True})

        is_true(t.c.x.compare(x_a))
        is_true(x_b.compare(x_a))

        is_true(x_a.compare(t.c.x))
        is_false(x_a.compare(t.c.y))
        is_false(t.c.y.compare(x_a))
        is_true((t.c.x == 5).compare(x_a == 5))
        is_false((t.c.y == 5).compare(x_a == 5))

        s = select(t).subquery()
        x_p = s.c.x
        is_false(x_a.compare(x_p))
        is_false(t.c.x.compare(x_p))
        x_p_a = x_p._annotate({})
        is_true(x_p_a.compare(x_p))
        is_true(x_p.compare(x_p_a))
        is_false(x_p_a.compare(x_a))

class ExecutableFlagsTest(fixtures.TestBase):
    @testing.combinations(
        (select(column("a")),),
        (table("q", column("a")).insert(),),
        (table("q", column("a")).update(),),
        (table("q", column("a")).delete(),),
        (lambda_stmt(lambda: select(column("a"))),),
    )
    def test_is_select(self, case):
        if isinstance(case, LambdaElement):
            resolved_case = case._resolved
        else:
            resolved_case = case

        if isinstance(resolved_case, Select):
            is_true(case.is_select)
        else:
            is_false(case.is_select)

class TypesTest(fixtures.TestBase):
    @testing.combinations(TypeDecorator, UserDefinedType)
    def test_thirdparty_no_cache(self, base):
        class MyType(base):
            impl = String

        expr = column("q", MyType()) == 1

        with expect_warnings(
            r"%s MyType\(\) will not produce a cache key" % base.__name__
        ):
            is_(expr._generate_cache_key(), None)

    @testing.combinations(TypeDecorator, UserDefinedType)
    def test_thirdparty_cache_false(self, base):
        class MyType(base):
            impl = String

            cache_ok = False

        expr = column("q", MyType()) == 1

        is_(expr._generate_cache_key(), None)

    @testing.combinations(TypeDecorator, UserDefinedType)
    def test_thirdparty_cache_ok(self, base):
        class MyType(base):
            impl = String

            cache_ok = True

        def go1():
            expr = column("q", MyType()) == 1
            return expr

        def go2():
            expr = column("p", MyType()) == 1
            return expr

        c1 = go1()._generate_cache_key()[0]
        c2 = go1()._generate_cache_key()[0]
        c3 = go2()._generate_cache_key()[0]

        eq_(c1, c2)
        ne_(c1, c3)

    def test_typedec_cache_ok_params(self):
        class MyType(TypeDecorator):
            impl = String

            cache_ok = True

            def __init__(self, p1, p2):
                self.p1 = p1
                self._p2 = p2

        def go1():
            expr = column("q", MyType("x", "y")) == 1
            return expr

        def go2():
            expr = column("q", MyType("q", "y")) == 1
            return expr

        def go3():
            expr = column("q", MyType("x", "z")) == 1
            return expr

        c1 = go1()._generate_cache_key()[0]
        c2 = go1()._generate_cache_key()[0]
        c3 = go2()._generate_cache_key()[0]
        c4 = go3()._generate_cache_key()[0]

        eq_(c1, c2)
        ne_(c1, c3)
        eq_(c1, c4)

    def test_thirdparty_sub_subclass_no_cache(self):
        class MyType(PickleType):
            pass

        expr = column("q", MyType()) == 1

        with expect_warnings(
            r"TypeDecorator MyType\(\) will not produce a cache key"
        ):
            is_(expr._generate_cache_key(), None)

    def test_userdefined_sub_subclass_no_cache(self):
        class MyType(UserDefinedType):
            cache_ok = True

        class MySubType(MyType):
            pass

        expr = column("q", MySubType()) == 1

        with expect_warnings(
            r"UserDefinedType MySubType\(\) will not produce a cache key"
        ):
            is_(expr._generate_cache_key(), None)

    def test_userdefined_sub_subclass_cache_ok(self):
        class MyType(UserDefinedType):
            cache_ok = True

        class MySubType(MyType):
            cache_ok = True

        def go1():
            expr = column("q", MySubType()) == 1
            return expr

        def go2():
            expr = column("p", MySubType()) == 1
            return expr

        c1 = go1()._generate_cache_key()[0]
        c2 = go1()._generate_cache_key()[0]
        c3 = go2()._generate_cache_key()[0]

        eq_(c1, c2)
        ne_(c1, c3)

    def test_thirdparty_sub_subclass_cache_ok(self):
        class MyType(PickleType):
            cache_ok = True

        def go1():
            expr = column("q", MyType()) == 1
            return expr

        def go2():
            expr = column("p", MyType()) == 1
            return expr

        c1 = go1()._generate_cache_key()[0]
        c2 = go1()._generate_cache_key()[0]
        c3 = go2()._generate_cache_key()[0]

        eq_(c1, c2)
        ne_(c1, c3)


import numpy as np

from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import LSTM
import string
import keras

# DONE: fill out the function below that transforms the input series
# and window-size into a set of input/output pairs for use with our RNN model
def window_transform_series(series, window_size):
    # containers for input/output pairs
    X = []
    y = []

    y = series[window_size:]
    series_size = len(series) - window_size
    i = 0
    while i < series_size:
        X.append(series[i: i + window_size])
        i = i + 1

    # reshape each
    X = np.asarray(X)
    X.shape = (np.shape(X)[0:2])
    y = np.asarray(y)
    y.shape = (len(y),1)

    return X,y

# DONE: build an RNN to perform regression on our time series input/output data
def build_part1_RNN(window_size):
    model = Sequential()
    model.add(LSTM(5, input_shape=(window_size, 1)))
    model.add(Dense(1))
    return model

# DONE: return the text input with only ascii lowercase and the punctuation given below included.
def cleaned_text(text):
    punctuation = [' ', '!', ',', '.', ':', ';', '?']
    text = text.lower()
    text_unique_characters = ''.join(set(text))
    for c in text_unique_characters:
        if c not in string.ascii_lowercase and c not in punctuation:
            text = text.replace(c, '')

    return text

# DONE: fill out the function below that transforms the input text and window-size into a set of input/output pairs for use with our RNN model
def window_transform_text(text, window_size, step_size):
    # containers for input/output pairs
    inputs = []
    outputs = []
    i = 0
    start_index = 0
    while start_index < len(text) - window_size:
        inputs.append(text[start_index: start_index + window_size])
        outputs.append(text[start_index + window_size: start_index + window_size + 1])
        i = i + 1
        start_index = i * step_size
    return inputs, outputs

# DONE build the required RNN model:
# a single LSTM hidden layer with softmax activation, categorical_crossentropy loss
def build_part2_RNN(window_size, num_chars):
    model = Sequential()
    model.add(LSTM(200, input_shape=(window_size,num_chars)))
    model.add(Dense(num_chars))
    model.add(Activation('softmax'))
    return model


from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render

from .models import Swell, Tide, SurfSession, SurfSpot
from datetime import datetime, timedelta, timezone

from .forms import AddSessionForm, AddSurfSpot, SessionMatchesConditions, SessionMatchesTimeAndPlace

# adding a session
def index(request):
    return render(request, 'surfinfo/index.html')

# adding a session
def add_session(request):
    # if this is a POST request, process form data
    if request.method == 'POST':
        form = AddSessionForm(request.POST)

        if form.is_valid():
            # process the data
            # grab swell info from Surfline API (do this first to get the surf spot UTC offset)
            surfSpotId = form.cleaned_data['surfSpot'].split(':')[1]
            surfSpotName = form.cleaned_data['surfSpot'].split(':')[0]
            startTime = form.cleaned_data['startTime']
            endTime = form.cleaned_data['endTime']
            surfScore = form.cleaned_data['surfScore']
            crowdScore = form.cleaned_data['crowdScore']
            waveCount = -1

            # only process the data if there was a spot selected
            if surfSpotId != 'NONE':
                # create a SurfSession object, save to DB
                todaySession = SurfSession.from_surfline(spotId=surfSpotId,
                                                         spotName=surfSpotName,
                                                         startTime=startTime,
                                                         endTime=endTime,
                                                         surfScore=surfScore,
                                                         crowdScore=crowdScore,
                                                         waveCount=waveCount)
                # redirect to a new URL
                return HttpResponseRedirect('congratsbro?sessionid=' + str(todaySession.id))
            else:
                return HttpResponseRedirect('newspotbro')

    # if this is not a POST request, create the form
    else:
        form = AddSessionForm()

    return render(request, 'surfinfo/sessionform.html', {'form': form})

# thank you for adding a session
def session_thankyou(request):
    if request.GET['sessionid']:
        if SurfSession.objects.filter(id=request.GET['sessionid']).exists():
            todaySession = SurfSession.objects.get(id=request.GET['sessionid'])
        else:
            todaySession = None
    else:
        todaySession = None

    # note for future: might need better conversion than typecasting as int for surfsession.spotUtcOffset from decimal to hours + minutes
    return render(request, 'surfinfo/sessionthankyou.html', {'surfsession': todaySession, 'surftimezone': timezone(
        offset=timedelta(hours=int(todaySession.spotUtcOffset)))})

# request a new surf spot
def request_new_spot(request):
    # if this is a POST request, process form data
    if request.method == 'POST':
        form = AddSurfSpot(request.POST)

        if form.is_valid():
            # process the data
            newSpot = SurfSpot(spotName=form.cleaned_data['spotName'], surflineId="TBD")
            newSpot.save()

            # redirect to a thank you URL
            return HttpResponseRedirect('thanksbro?spotid=' + str(newSpot.id))
    # if this is not a POST request, create the form
    else:
        form = AddSurfSpot()

    return render(request, 'surfinfo/surfspotform.html', {'form': form})

# thank the user for requesting a new surf spot
def spot_thankyou(request):
    spotid = request.GET.get('spotid')

    if spotid:
        if SurfSpot.objects.filter(id=request.GET['spotid']).exists():
            newSpot = SurfSpot.objects.get(id=request.GET['spotid'])
        else:
            newSpot = None
    else:
        newSpot = None

    return render(request, 'surfinfo/spotthankyou.html', {'newSpot': newSpot})

# find matching sessions for surf conditions
def session_matches_conditions(request):
    form = SessionMatchesConditions()

    tide = request.GET.get('tideHeight', 0)

    swell = Swell(height=request.GET.get('swellHeight', 0),
                  period=request.GET.get('swellPeriod', 0),
                  direction=request.GET.get('swellDirection', 0))

    sessions = SurfSession.get_matching_sessions(swellHeight=swell.height,
                                                 swellPeriod=swell.period,
                                                 swellDirection=swell.direction,
                                                 tideHeight=tide,
                                                 processedFlag=True)

    return render(request, 'surfinfo/getmatchingsessionsummary.html',
                  {'form': form,
                   'swellHeight': swell.height,
                   'swellPeriod': swell.period,
                   'swellDirection': swell.direction,
                   'swellPower': swell.power,
                   'tideHeight': tide,
                   'sessions': sessions})

# show full detailed view of all matching sessions for a set of conditions and a spot
# find matching sessions for surf conditions
def full_list_session_matches_conditions(request):
    tide = request.GET.get('tideHeight', 0)

    swell = Swell(height=request.GET.get('swellHeight', 0),
                  period=request.GET.get('swellPeriod', 0),
                  direction=request.GET.get('swellDirection', 0))

    sessions = SurfSession.get_matching_sessions(swellHeight=swell.height,
                                                 swellPeriod=swell.period,
                                                 swellDirection=swell.direction,
                                                 tideHeight=tide,
                                                 processedFlag=False)

    surftimezone = timezone(offset=timedelta(hours=int(sessions[0].spotUtcOffset)))

    return render(request, 'surfinfo/getmatchingsessionlist.html',
                  {'swellHeight': swell.height,
                   'swellPeriod': swell.period,
                   'swellDirection': swell.direction,
                   'swellPower': swell.power,
                   'tideHeight': tide,
                   'sessions': sessions,
                   'surftimezone': surftimezone,
                   })

# find matching sessions for conditions at a given time and region
def session_matches_time_and_place(request):
    surfDatetime = request.GET.get('surfDatetime', None)
    surfRegion = request.GET.get('surfRegion', None)

    REGION_TO_SPOT = {'58581a836630e24c44879011': '5842041f4e65fad6a7708805',
                      '5cc73566c30e4c0001096989': '5842041f4e65fad6a7708976',
                      '58581a836630e24c44879010': '5842041f4e65fad6a77087f8',
                      '58581a836630e24c44878fd7': '5842041f4e65fad6a77088b3',
                      '58581a836630e24c4487900d': '5842041f4e65fad6a770883b',
                      }

    form = SessionMatchesTimeAndPlace()
    displayDatetime = None
    swells = []
    tide = []

    if surfDatetime is not None and surfRegion is not None:
        surfDatetime = datetime.strptime(surfDatetime, '%Y-%m-%dT%H:%M')
        displayDatetime = surfDatetime.strftime('%Y-%m-%dT%H:%M')

        swells = Swell.get_surfline_swells(surflineId=surfRegion,
                                           subregionFlag=True,
                                           surfDatetime=surfDatetime)

        tide = Tide.get_surfline_tides(surflineId=REGION_TO_SPOT[surfRegion],
                                       startDatetime=surfDatetime,
                                       endDatetime=surfDatetime + timedelta(milliseconds=1))

    return render(request, 'surfinfo/getconditions.html',
                  {'form': form,
                   'surfDatetime': displayDatetime,
                   'surfRegion': surfRegion,
                   'swells': swells,
                   'junk_threshold': Swell.JUNK_THRESHOLD,
                   'tide': next(iter(tide), None)})

# -----------------------------------------------------------------------------------
# bootstrap DB with historical data
def data_bootstrap(request):
    sessionId = request.GET.get('id', 0)

    if int(sessionId) > 0:
        return HttpResponse(SurfSession.data_extract(sessionId))
    else:
        if SurfSession.data_bootstrap():
            return HttpResponse("All done bro, historical sessions locked and loaded")
        else:
            return HttpResponse("Already bootstrapped bro, ADIOS MOTHAFUCKA")


import datetime as dt
from datetime import timedelta
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.operators.python_operator import PythonOperator

def greet(r_date):
    print('Writing in file')
    with open('./greet.txt', 'a+', encoding='utf8') as f:
        now = dt.datetime.now()
        f.write(str(r_date) + '\n')
    return 'Greeted'
def respond():
    return 'Greet Responded Again'

default_args = {
    'owner': 'airflow',
    'start_date': dt.datetime(2020, 6, 2, 12, 30, 00),
    'concurrency': 1,
    'depends_on_past': True,
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(seconds=10),

}

with DAG('simple_dag', default_args=default_args, schedule_interval='*/1 * * * *')as dag:
    opr_hello = BashOperator(task_id='say_hi',
                             bash_command='echo Hi !!')
    opr_greet = PythonOperator(task_id='greet',
                               python_callable=greet,
                               op_kwargs={'r_date':'{{ (execution_date - macros.timedelta(days=5)).strftime("%Y-%m-%d") }} '})
    opr_sleep = BashOperator(task_id='sleep_me',
                             bash_command='sleep 2')
    opr_respond = PythonOperator(task_id='respond',
                                 python_callable=respond)

    opr_hello >> opr_greet >> opr_sleep >> opr_respond


import random
import time
import queue
import hashlib
from threading import *
from treelib import Node, Tree
from datetime import timedelta
import threading

lock = Lock() # Global LOCK (use to mutual exclusion)

n = 3 # Network cardinality (number of processes in the network)

growth_speed = 0.1 # regulates the growth rate of process blockchains

global_Δ = 2 # global time parameter Δ

# BROADCAST COMMUNICATION ABSTRACTION

# Channel of PROPOSAL messages
broadcast_proposal = {
		"1" : queue.Queue(),
        "2" : queue.Queue(),
        "3" : queue.Queue()
}

# Channel of VOTE messages
broadcast_vote = {
		"1" : queue.Queue(),
        "2" : queue.Queue(),
        "3" : queue.Queue()
}

# Channel of CLOCK messages
broadcast_clock = {
		"1" : queue.Queue(),
        "2" : queue.Queue(),
        "3" : queue.Queue()
}

# Given an epoch e, it randomly returns the proponent of this epoch (deterministically)
def proposer(e):
    random.seed(e)
    return str(random.randint(1, n)) # It's possible to change this implementation of random selection

# Returns an hash of a single Process identifier together with a Block object
def sign_block(process, block):
    signature = (process.identifier, block)
    return hash(signature)

# Returns an hash of a single Process identifier together with a Clock object
def sign_clock(process, clock):
    signature = (process.identifier, clock)
    return hash(signature)

def vrfy_block(process, block, signature):
    if sign_block(process,block) == signature:
        return True
    else:
        return False

# Class representing a single block
class Block:
    def __init__(self, epoch, TXs, hash=0):
        self.epoch = epoch # Block's local epoch
        self.TXs = TXs # Transactions
        self.hash = hash # Previous block hash (recursive reference to the chain)

# Class representing an entire tree of blocks
class BlockChain:
    def __init__(self):
        self.root = Tree()
        self.root.create_node(0, 0) # Genesis block

    # Adds a block to a blockchain
    def add_block(self, block):
        node = self.root.create_node(block.epoch, hash(block), block.hash)
        node.data = block

    # Print the blockchain graphically
    def print_chain(self):
        self.root.show()

    # Returns the list containing all the blocks of the entire blockchain
    def return_nodes_data(self):
        nodes = self.root.all_nodes()
        nodes_data = []
        for i in range(1,len(nodes)):
            nodes_data.append(nodes[i].data)
        if len(nodes_data) == 0:
            nodes_data.append(0)
        return nodes_data

    # Returns the list of identifiers of the leaves of the tree (blockchain)
    def leaves(self):
        leaves = self.root.leaves(nid=None)
        leaves_identifiers = []
        for i in range(len(leaves)):
            leaves_identifiers.append(leaves[i].identifier)
        return leaves_identifiers

    # returns the Block object with the major epoch in the blockchain
    def block_max_epoch(self):
        nodes = self.root.all_nodes()
        max_epoch = 0
        block_max = 0
        for i in range(len(nodes)):
            if nodes[i].tag > max_epoch:
                max_epoch = nodes[i].tag
                block_max = nodes[i].data
        return block_max

# Proposal message object
class Proposal:
    def __init__(self, sender, block, signature):
        self.sender = sender
        self.block = block
        self.signature = signature

# Vote message object
class Vote:
    def __init__(self, sender, block, signature):
        self.sender = sender
        self.block = block
        self.signature = signature

# Clock message object
class Clock:
    def __init__(self, sender, epoch, signature):
        self.sender = sender
        self.epoch = epoch
        self.signature = signature

# Class representing a single process
class Process(Thread):
    def __init__(self, identifier):
        super(Process, self).__init__()

        self.B = BlockChain() # Process blockchain, initialized to Genesis Block
        self.e = 1 # Number of local epoch
        self.has_voted = [False for i in range(200)]
        self.prop_rec = ['⊥' for i in range(200)]
        self.fchain = ['⊥' for i in range(200)] # Output: current finalized chain

        self.identifier = identifier
        self.TX = 'Transactions pool' # It simulates an entire transaction pool of a process

        self.Δ = global_Δ # Local time clock for epoch change
        self.votes = {} # Vote counter for each block
        self.clocks = {} # Clock counter for each epoch

    # Time passing simulator: every second the local Δ is decreased by 1
    def countdown(self):
        while self.Δ > 0:
            self.Δ = self.Δ - 1
            time.sleep(1)

    # Update the number of votes for each Block object that arrives
    def counter_votes(self):
        while True:
            vote = broadcast_vote[self.identifier].get() # It waits on the queue
            if vote.block not in self.votes:
                self.votes[vote.block] = 1
            else:
                self.votes[vote.block] = self.votes[vote.block] + 1 # aumenta il contatore di voti relativo a un blocco

            # TODO: FLOODING

            time.sleep(growth_speed)

    # Update the number of clocks for each Epoch object that arrives
    def counter_clocks(self):
        while True:
            clock = broadcast_clock[self.identifier].get() # It waits on the queue
            if clock.epoch not in self.clocks:
                self.clocks[clock.epoch] = 1
            else:
                self.clocks[clock.epoch] = self.clocks[clock.epoch] + 1

            # TODO: FLOODING

            time.sleep(growth_speed)

    '''
    EVENT 1: trigger when the value of 'e' changes
    '''
    def New_epoch_start(self):
        e_prec = 0
        while True:
            if self.identifier == proposer(self.e) and e_prec != self.e: # se io sono il proponente dell'epoca

                e_prec = self.e # to not repeat the function twice

                lb = self.B.block_max_epoch() # lb is the "last block"

                b = Block(self.e, self.TX, hash(lb)) # creation of a new block b

                my_proposal = Proposal(self, b, sign_block(self, b)) # create the Proposal object

                # Proposal object broadcast
                for k in broadcast_proposal:
                    # time.sleep(random.uniform(0.0, 3.0)) # It simulates different speeds in the network
                    broadcast_proposal[k].put(my_proposal)

            time.sleep(growth_speed)

    '''
    EVENT 2: acceptance of a Block proposal (first and only time!)
    '''
    def New_proposal(self):

        while True:
            proposal = broadcast_proposal[self.identifier].get() # It waits on the queue

            if vrfy_block(proposal.sender, proposal.block, proposal.signature) and proposal.sender.identifier == proposer(proposal.block.epoch) and self.prop_rec[proposal.block.epoch] == '⊥' :
                self.prop_rec[proposal.block.epoch] = proposal.block

            time.sleep(growth_speed)

    '''
    EVENT 3: counter of votes for blockchain update and change of epoch
    '''
    def Update_blockchain(self):
        while True:
            for b in self.votes:
                if self.votes[b] >= 2*n/3:

                    # TODO: aggiungere un meccanismo per controllare le firme

                    self.B.add_block(b)
                    self.votes[b] = -1

                    self.finalize()

                    self.e = max(self.e , b.epoch + 1) # update epoch
                    self.Δ = global_Δ # update Δ for waiting for the next age

            time.sleep(growth_speed)

    '''
    EVENT 4: broadcast of the clock message for possible epoch update
    '''
    def Clock_broadcast(self):
        while True:
            if self.Δ == 0:
                print('cloook')
                my_clock = Clock(self, self.e, sign_clock(self,self.e)) # Clock message object
                for k in broadcast_clock:
                    # time.sleep(random.uniform(0.0, 3.0)) # It simulates different speeds in the network
                    broadcast_clock[k].put(my_clock)
                self.Δ = -1 # To prevent it from continuing to send messages
            time.sleep(growth_speed)

    '''
    EVENT 5: clock counter for possible epoch update
    '''
    def Change_epoch(self):
        while True:
            for e in self.clocks:
                if self.clocks[e] >= 2*n/3:
                    # TODO: aggiungere un meccanismo per controllare le firme
                    self.e = max(self.e, e + 1) # Update current epoch
                    self.Δ = global_Δ # update Δ for waiting for the next age

            time.sleep(growth_speed)

    '''
    EVENT 6: Vote event
    '''
    def Vote(self):

        while True:
            b_parent = None

            for block in self.B.return_nodes_data():
                if self.prop_rec[self.e] != '⊥' and hash(block) == self.prop_rec[self.e].hash:
                    if self.B.block_max_epoch() == block:
                        b_parent = block
                        break

            if self.prop_rec[self.e] != '⊥' and self.has_voted[self.e] == False and b_parent != None:
                b = self.prop_rec[self.e]
                my_vote = Vote(self, b, sign_block(self,b))
                for k in broadcast_vote:
                    time.sleep(random.uniform(0.0, 7.0)) # It simulates different speeds in the network
                    broadcast_vote[k].put(my_vote)
                self.has_voted[self.e] = True

            time.sleep(growth_speed)

    # Updates the finalized chain
    def finalize(self):

        '''
        N = set() # N is the set of "normal blocks"
        for b in self.B.return_nodes_data(): # b is a block
            for b_1 in self.B.return_nodes_data(): # b_1 is a block
                if b.epoch + 1 == b_1.epoch and hash(b) == b_1.hash:
                    N.add(b)

        max_epoch = 0
        lnb = None # Last 'normal block'
        for b in N:
                if b.epoch >= max_epoch:
                    max_epoch = b.epoch
                    lnb = b

        self.fchain[max_epoch] = lnb

        '''

    def print_demonstration(self):
        while True:
            with lock:

                print('.............')
                print('blockchain of ' + self.identifier + ' (epoch ' + str(self.e) + '):')
                if proposer(self.e) == self.identifier:
                    print('PROPOSER')
                self.B.print_chain()
                print('.............')

            time.sleep(1)

    def run(self):

        # Sotto-thread (contatori e altro)
        p1 = Thread(target = self.countdown)
        p2 = Thread(target = self.counter_votes)
        p3 = Thread(target = self.counter_clocks)

        # Sotto-thread (EVENTI)
        e1 = Thread(target = self.New_epoch_start)
        e2 = Thread(target = self.New_proposal)
        e3 = Thread(target = self.Update_blockchain)
        e4 = Thread(target = self.Clock_broadcast)
        e5 = Thread(target = self.Change_epoch)
        e6 = Thread(target = self.Vote)

        p1.start()
        p2.start()
        p3.start()
        e1.start()
        e2.start()
        e3.start()
        e4.start()
        e5.start()
        e6.start()

        print_demonstration = Thread(target = self.print_demonstration)
        print_demonstration.start()

def main():

    process_1 = Process("1")
    process_1.start()

    process_2 = Process("2")
    process_2.start()

    process_3 = Process("3")
    process_3.start()

if __name__ == '__main__':
    main();


import sys,re,difflib,time,datetime
import numpy as np
from array import array
np.set_printoptions(threshold=sys.maxsize)
import matplotlib.pyplot as plt
import networkx as nx

def show_graph_with_labels(adjacency_matrix, mylabels):
    rows, cols = np.where(adjacency_matrix != 0)
    edges = zip(rows.tolist(), cols.tolist())
    gr = nx.Graph()
    gr.add_edges_from(edges)
    pos=nx.spring_layout(gr)
    nx.draw(gr,pos, node_size=500, with_labels=True, edge_color='b')
    plt.show()

def make_label(labels):
    l = {}
    for i, label in enumerate(labels):
        l[i] = label
    return l

P = np.loadtxt("test2")
show_graph_with_labels(P,make_label(range(len(P)+10)))


from google.cloud import aiplatform
import numpy as np

endpoint_name = "text-classification_endpoint"
endpoint = aiplatform.Endpoint.list(filter=f'display_name="{endpoint_name}"')[0]
text = "I love going to the movies!"
result = endpoint.predict(instances=[{'content': text}])
prediction = result.predictions[0]
names = prediction['displayNames']
scores = prediction['confidences']
print(
    f"Sentiment is predicted as `{names[np.argmax(scores)]}` with a score of {round(100*max(scores), 2)}%"
)


import fnmatch
import os

import PyPDF2
import openpyxl
from PyPDF2 import PdfReader
from openpyxl.styles import Border, Side, Font, Alignment
from openpyxl.worksheet.worksheet import Worksheet

excel_path = os.getcwd()
listOfFiles = os.listdir('.')

pdf_files = [f for f in listOfFiles if f.endswith('.pdf')]
pdf_files.sort()

pdf_path = []
for root, dirs, files in os.walk(excel_path):
    for file in files:
        if not file.endswith(".pdf"):
            continue
        pdf_path.append(os.path.join(root, file))
        print(pdf_path)

r = p = k = 11
# p = 11
l = 0
number = 1
k = 11
ob_list = []
naim_list = []
count_list = []

for name in pdf_files:
    global ob
    global naim
    # global name
    global sb
    name = name.split('.pdf')  # отделяем формат от имени файла
    name1 = name.pop(0)  # выбираем первый элемент из списка
    name = name1.split(' ')  # отделяем пробелы
    ob = name.pop(0)  # выбираем в качестве обозначения первый элемент списка
    ob = ob.split('БТЛИ.')  # отделяем от обозначения БТЛИ
    ob = ob.pop(1)  # выбираем второй элемент списка
    if 'СБ' in name:  # проверка на присутствие СБ в названии
        # sb = name.pop(1)
        sb = 'СБ'
        ob = ob + sb
        naim = name.pop(1)
    else:
        sb = ' '
        ob = ob
        naim = name.pop(0)
    count_big_naim = len(sb)
    if count_big_naim > 1:
        global naim2
        naim2 = " ".join(name)
        # naim3 = naim2.split('СБ ')
        # naim = [' '.join(naim3)]
    ob_list.append(ob)  # пополнить список обозначений
    naim_list.append(naim)  # пополнить список наименований

    # чтение PDF
    reader = PdfReader(pdf_files[l])
    number_of_pages = len(reader.pages)
    page = reader.pages[0]

    count_list.append(number_of_pages)
    l += 1
    # чтение размеров листа
    sizes = []
    pdf = 0
    # print(pdf_str)
    for pdf in pdf_path:
        with open(pdf, 'rb') as pdf_file:
            pdf_reader = PyPDF2.PdfFileReader(pdf_file)
            i = 0
            while True:
                try:
                    box = pdf_reader.getPage(i)
                    width = int(float(box.mediaBox.getWidth()) * 0.3527)
                    height = int(float(box.mediaBox.getHeight()) * 0.3527)
                    sizes.append([height, width])
                    i += 1
                except Exception:
                    break

print(sizes)
wb = openpyxl.load_workbook("Служебная записка на обработку и размножение чертежей.xlsx")
sheets = wb.sheetnames
sheet_ranges = wb['Лист1']
ws: Worksheet = wb.active
ws.title = 'Лист1'

string_number = len(ob_list)

# Параметры стиля ячеек
thin_border = Border(left=Side(style='thin'),
                     right=Side(style='thin'),
                     top=Side(style='thin'),
                     bottom=Side(style='thin'))
# Параметры шрифта
font = Font(name='Times New Roman',
            size=12,
            color='FF000000',
            bold=False,
            italic=False,
            vertAlign=None,
            underline='none',
            strike=False)

# Параметры выравнивания
alignment = Alignment(horizontal='center',
                      vertical='center',
                      text_rotation=0,
                      wrap_text=False,
                      shrink_to_fit=False,
                      indent=0)

for row in ob_list:
    ws.cell(row=r, column=10).value = row
    ws.cell(row=r, column=10).border = thin_border
    ws.cell(row=r, column=10).font = font
    ws.cell(row=r, column=10).alignment = alignment
    r += 1
    ws.insert_rows(r + 1)
for row in naim_list:
    ws.cell(row=p, column=11).value = row
    ws.cell(row=p, column=11).border = thin_border
    ws.cell(row=p, column=11).font = font
    ws.cell(row=p, column=11).alignment = alignment
    p += 1
for row in count_list:
    ws.cell(row=k, column=12).value = row
    ws.cell(row=k, column=12).font = font
    ws.cell(row=k, column=12).alignment = alignment
    ws.cell(row=k, column=12).border = thin_border
    ws.cell(row=k, column=1).value = number
    ws.cell(row=k, column=1).font = font
    ws.cell(row=k, column=1).alignment = alignment
    ws.cell(row=k, column=1).border = thin_border
    ws.cell(row=k, column=6).border = thin_border
    ws.cell(row=k, column=7).border = thin_border
    ws.cell(row=k, column=8).border = thin_border
    ws.cell(row=k, column=13).border = thin_border
#  ввод БТЛИ
    ws.cell(row=k, column=9).value = 'БТЛИ.'
    ws.cell(row=k, column=9).font = font
    ws.cell(row=k, column=9).alignment = alignment
    ws.cell(row=k, column=9).border = thin_border
    k += 1
    number += 1

# Объединить ячейки
def merge():
    ws.merge_cells(start_row=11, start_column=2, end_column=2, end_row=k - 1)
    ws.merge_cells(start_row=11, start_column=3, end_column=3, end_row=k - 1)
    ws.merge_cells(start_row=11, start_column=4, end_column=4, end_row=k - 1)
    ws.merge_cells(start_row=11, start_column=5, end_column=5, end_row=k - 1)

merge()

wb.save('Служебная записка на обработку и размножение чертежей.xlsx')

print(count_list)
print(ob_list)
print(naim_list)
print(naim)
print(naim2)

Заполнение служебной записки/zap.py


# -*- coding: utf-8 -*-
"""
Created on Thu Jun 21 15:42:31 2018

@author: Ashish Chouhan
"""

#The optimal values of m and b can be actually calculated with way less effort than doing a linear regression.
#this is just to demonstrate gradient descent

import numpy as np
from matplotlib import pyplot as plt

# y = mx + b
# m is slope, b is y-intercept

def plot_lines(m,b,data_points):
    x_values = [i for i in range(int(min(data_points))-1, int(max(data_points))+2)]
    y_values = [(m*x + b) for x in x_values]
    plt.plot(x_values, y_values, 'r')

def compute_error_for_line_given_points(b, m, points):
    totalError = 0
    for i in range(0, len(points)):
        x = points[i, 0]
        y = points[i, 1]
        totalError += (y - (m * x + b)) ** 2
    return totalError / float(len(points))

def step_gradient(b_current, m_current, points, learningRate):
    b_gradient = 0
    m_gradient = 0
    N = float(len(points))
    for i in range(0, len(points)):
        x = points[i, 0]
        y = points[i, 1]
        b_gradient += (1/N) * (((m_current * x) + b_current) - y)
        m_gradient += +(1/N) * x * (((m_current * x) + b_current) - y)
    new_b = b_current - (learningRate * b_gradient)
    new_m = m_current - (learningRate * m_gradient)
    return [new_b, new_m]

def gradient_descent_runner(points, starting_b, starting_m, learning_rate, num_iterations):
    b = starting_b
    m = starting_m
    a= []

    for i in range(0, len(points)):
        a.append(points[i,0])

    data_points = np.array(a).tolist()

    for i in range(num_iterations):
        b, m = step_gradient(b, m, np.array(points), learning_rate)

        for i in range(0, len(points)):
            x = points[i, 0]
            y = points[i, 1]
            plt.plot(x,y,'bo')

        plot_lines(m,b,data_points)

    return [b, m]

def run():
    points = np.genfromtxt("anscombe.csv", delimiter=",")

    learning_rate = 0.001
    initial_b = 0 # initial y-intercept guess
    initial_m = 0 # initial slope guess
    num_iterations = 50
    print ("Starting gradient descent at b = {0}, m = {1}, error = {2}".format(initial_b, initial_m, compute_error_for_line_given_points(initial_b, initial_m, points)))
    print ("Running...")
    [b, m] = gradient_descent_runner(points, initial_b, initial_m, learning_rate, num_iterations)
    print ("After {0} iterations b = {1}, m = {2}, error = {3}".format(num_iterations, b, m, compute_error_for_line_given_points(b, m, points)))

if __name__ == '__main__':
    run()

Linear Regression/GradientDescent_With Plot Feature.py