# Implements SynthesisGraph, a data-structure that encodes results of multiple
# synthesis tasks

import argparse
import tempfile as tf
from collections import defaultdict
import re
import uuid
import itertools

from loguru import logger

import numpy as np
import pandas as pd

import networkx as nx
import yaml

from synthesis.work_queue import BeanstalkWQ, ListWQ
from synthesis.task_result import SynthesisTaskResult

class SynthesisGraph:
    # Define some colors
    unsafe_color = "red"
    safe_color = "green"
    success_color = "cadetblue2"

    def __init__(self):
        self.sg = nx.MultiDiGraph()
        self.node_map = dict()
        self.ctr = -1
        self.stats = defaultdict(lambda: 0)
        self.programs = defaultdict(list)

    def add_node(self, opcode):
        opcode = opcode.upper()
        if opcode in self.node_map:
            return self.node_map[opcode]

        self.ctr += 1

        self.sg.add_node(self.ctr, label=opcode, data=opcode)
        self.node_map[opcode] = self.ctr

        return self.ctr

    def set_node_safe(self, node):
        self.sg.nodes[node]["state"] = "safe"

    def set_node_unsafe(self, node):
        self.sg.nodes[node]["state"] = "unsafe"

    def set_node_synthesizeable(self, node):
        self.sg.nodes[node]["state"] = "synth"

    def iter_nodes(self):
        return self.sg.nodes()

    def iter_solutions(self, node):
        solutions = set()
        for edge in self.sg.out_edges(node, data=True):
            solutions.add(edge[2]["tag"])

        return list(solutions)

    def iter_solution(self, node, solution):
        nexts = []
        for edge in self.sg.out_edges(node, data=True):
            if edge[2]["tag"] != solution:
                continue
            nexts.append((edge[1], edge[2]["lineno"], edge[2]["ops"]))

        return [(x[0], x[2]) for x in sorted(nexts, key=lambda y: y[1])]

    def discard_solution(self, node, solution):
        edges_to_remove = []

        for edge in self.sg.edges(node, keys=True, data=True):
            if edge[3]["tag"] == solution:
                edges_to_remove.append((edge[0], edge[1], edge[2]))

        for p, c, i in edges_to_remove:
            self.sg.remove_edge(p, c, key=i)

    def iter_users(self, node):
        return [x[0] for x in self.sg.in_edges(node)]

    def inst(self, node):
        return self.sg.nodes[node]["data"]

    def get_safe_sets(self):
        from sympy import symbols
        from functools import reduce
        import operator
        from sympy.logic import simplify_logic
        from sympy.logic.boolalg import to_cnf, to_dnf

        vars = dict()
        expressions = dict()
        for node in self.sg.nodes():
            vars[node] = symbols(self.inst(node))

        for node in self.sg.nodes():
            exprs = []
            for sol in self.iter_solutions(node):
                expr = None
                for nidx, _ in self.iter_solution(node, sol):
                    if expr:
                        expr = expr & vars[nidx]
                    else:
                        expr = vars[nidx]
                exprs.append(expr)
            exprs.append(vars[node])
            expressions[node] = reduce(operator.or_, exprs)

        final_expression = reduce(operator.and_, expressions.values())
        print(to_dnf(final_expression, simplify=True, force=True))

    def get_safe_sets_nnf(self):
        from nnf import Var, Or, And

        big_vars = dict()
        small_vars = dict()
        expressions = dict()

        source_node = None
        for node in self.sg.nodes():
            inst = self.inst(node)
            if inst == 'ORQ-R64-R64':
                source_node = node
                break

        small_graph = nx.induced_subgraph(
            self.sg,
            [x for x in nx.dfs_tree(self.sg, source=source_node, depth_limit=2)]).copy()
        self.sg = small_graph
        self.to_svg("SMALL-GRAPH.svg")

        for node in self.sg.nodes():
            big_vars[node] = Var(self.inst(node))
            small_vars[node] = Var(self.inst(node).lower())

        for node in self.sg.nodes():
            exprs = []
            for sol in self.iter_solutions(node):
                expr = And([big_vars[nidx] for nidx, _ in self.iter_solution(node, sol)])
                exprs.append(expr)

            exprs.append(small_vars[node])
            expressions[node] = Or(exprs)

        final_expression = And(expressions.values())

        logger.debug(f"Expression: {final_expression}")
        logger.debug(f"Implicants: {final_expression.implicants()}")

    def get_safe_sets_brute(self):
        sccs = nx.strongly_connected_components(self.sg)
        sccs = [scc for scc in sccs if len(scc) > 1]

        accepted_sets = []
        for scc in sccs:
            scc_sg = SynthesisGraph()
            scc_g = nx.induced_subgraph(self.sg, scc).copy()
            scc_sg.sg = scc_g

            names = [self.inst(n) for n in scc]
            if all(map(lambda x: x.startswith("PSEUDO-"), names)):
                self.sg.remove_nodes_from(scc)
                continue

            next_round = list(scc)
            current_round = 1

            this_scc_safe = []

            while current_round <= len(next_round):
                pot_safe_sets = [
                    [x] for x in itertools.combinations(next_round, current_round)
                ]
                next_round = set()
                while pot_safe_sets:
                    pot_set = pot_safe_sets.pop(0)
                    can_implement = scc_sg._is_sufficient_safe_set(pot_set)
                    if can_implement:
                        this_scc_safe.append(pot_set)
                    else:
                        next_round.update(pot_set)
                current_round += 1

            accepted_sets.append(this_scc_safe)

        core_set = []
        for node in self.sg.nodes():
            solutions = list(self.iter_solutions(node))
            if not solutions:
                core_set.append(node)

    def _is_sufficient_safe_set(self, safe_set):
        visited = list()
        wl = list(safe_set)

        while wl:
            current = wl.pop(0)
            if current in visited:
                continue

            for sol in self.iter_solutions(current):
                can_implement = True
                for ni, _ in self.iter_solution(current, sol):
                    if ni not in visited:
                        can_implement = False
                        break

                if can_implement or current in safe_set:
                    for user in self.iter_users(current):
                        if user not in wl and user not in visited:
                            wl.append(user)
                    visited.append(current)
                    break

        all_nodes = self.sg.nodes()
        return all(map(lambda n: n in visited, all_nodes))

    def append_result(self, synth_result):
        self.stats[synth_result.state] += 1

        # Add failed spec to graph as well
        spec = synth_result.spec
        specid = self.add_node(spec)

        if not synth_result.is_success():
            return

        parsed_prog = self._parse_program(synth_result.program)

        if self.is_redundant_solution(spec, parsed_prog):
            self.stats["redundant"] += 1
            return

        # Save to programs
        self.programs[spec].append(parsed_prog)

        for idx, line in enumerate(parsed_prog):
            opcode = line["opcode"]
            opcodeid = self.add_node(opcode)

            tag = self._name_to_tag(synth_result.name)
            logger.info(f"{spec} -> {opcode}")

            self.sg.add_edge(
                specid,
                opcodeid,
                tag=tag,
                lineno=idx,
                ops=line["ops"],
                label=f'({tag}, {idx}, [{",".join(line["ops"])}])',
            )

        # self.to_svg()

    def is_redundant_solution(self, spec, parsed_prog):
        is_seen_solution = []
        for idx, prog in enumerate(self.programs[spec]):
            is_seen_solution.append(True)

            if len(prog) != len(parsed_prog):
                is_seen_solution[idx] = False
                continue

            for p, n in zip(prog, parsed_prog):
                if p["opcode"] != n["opcode"]:
                    is_seen_solution[idx] = False
                    break

        if is_seen_solution and any(is_seen_solution):
            return True

        return False

    #################################################################
    ### Helpers
    #################################################################

    def _parse_program(self, program):
        results = []
        lines = program.split("\n")[1:]

        for line in lines:
            if not line:
                continue

            result = dict()
            opcode = line.split(" ", 1)[0].upper()

            operands = []
            if "[" in line:
                operands = line.split("] ")
                if len(operands) > 1:
                    operands = operands[1].split(",")
                else:
                    operands = []
            elif not line.startswith("pnop"):
                operands = line.split(" ", 1)[1].split(",")

            result["opcode"] = opcode
            result["ops"] = operands

            results.append(result)

        return results

    def _name_to_tag(self, name):
        id = name.rsplit("-", 1)[1][:-4]
        return id

    ###############################################################
    ### Add cosmetic information to nodes/edges
    ###############################################################
    def color(self, node, color):
        self.sg.nodes[node]["fillcolor"] = color
        self.sg.nodes[node]["style"] = "filled"

    ################################################################
    ### Utilities to write/read data to/from files
    ################################################################

    def to_file(self, fname):
        nx.write_gpickle(self.sg, fname)

    def save_stats(self, fname):
        with open(fname, "w") as fd:
            fd.write(yaml.dump(self.stats))

        with open(f"{fname[:-5]}-progs.yaml", "w") as fd:
            fd.write(yaml.dump(self.programs))

    def load_partial_graph(self, gname, progs=None):
        logger.info(f"Loading graph: {gname}")
        self.sg = nx.read_gpickle(gname)

        # Populate the other fields
        for node in self.sg.nodes():
            label = self.sg.nodes[node]["label"]
            self.node_map[node] = label

        self.ctr = len(self.sg.nodes()) - 1

        if progs:
            logger.info(f"Loading progs: {progs}")
            # Reconstruct programs from progs
            with open(progs) as fd:
                self.progs = yaml.load(fd)

    def to_svg(self, fname=None):
        from matplotlib import pyplot as plt
        from networkx.drawing.nx_agraph import graphviz_layout, write_dot

        if not fname:
            fname = tf.NamedTemporaryFile(
                dir="synth-results/", prefix="synthg-", suffix=".svg", delete=False
            ).name

        logger.info(f"Writing to: {fname}")

        plt.figure(3, figsize=(12, 9))
        pos = graphviz_layout(self.sg, prog="dot", root=0, args="-Gsize=12,9")
        nx.draw(self.sg, pos, node_color="w", node_shape="s")
        node_labels = nx.get_node_attributes(self.sg, "data")

        for k, v in node_labels.items():
            node_labels[k] = f"{v}"

        nx.draw_networkx_labels(self.sg, pos, labels=node_labels, font_size=10)

        write_dot(self.sg, fname[:-4] + ".dot")

        plt.savefig(fname)
        plt.show()
        plt.close()

def do_push_back(wq, epochs, synth_result):
    epochs[synth_result.spec] += 1

    if epochs[synth_result.spec] < 32:
        wq.push(synth_result.spec)

def generate(results, wq, outfile=None, continue_partial=None):
    from tqdm import tqdm

    epochs = defaultdict(lambda: 0)
    sg = SynthesisGraph()

    if continue_partial:
        graph_path = continue_partial + ".pkl"
        progs = continue_partial + "-progs.yaml"
        sg.load_partial_graph(graph_path, progs)

    try:
        for res in results:
            st = SynthesisTaskResult.from_json(res)
            logger.info(f"Processing: {st}")
            do_push_back(wq, epochs, st)
            sg.append_result(st)
    except KeyboardInterrupt:
        logger.warning("Got keyboard interrupt; saving!")
    finally:
        # sg.save_stats(outfile + ".yaml")
        sg.to_file(outfile + ".pkl")
        logger.success("Generated .pkl file!")
        # sg.to_svg(outfile + ".svg")

    return sg

def stitch_flag_solutions(sg, flag_file):
    jresults = []
    with open(flag_file) as fd:
        results = yaml.load(fd)
    for name, res in results.items():
        jresults.append(res.__dict__)
    results = jresults

    args.o += "-flags"
    flags_sg = generate(results, wq, args.o, "")

    name_to_idx = dict()
    flag_to_idx = dict()

    for node in sg.sg.nodes():
        name = sg.sg.nodes[node]["label"]
        name_to_idx[name] = node

    for node in flags_sg.sg.nodes():
        name = flags_sg.sg.nodes[node]["label"]
        flag_to_idx[name] = node

    # Now, append flags results to the original synthesis graph
    for node in flags_sg.sg.nodes():
        name = flags_sg.sg.nodes[node]["label"]
        flag_solutions = list(flags_sg.iter_solutions(node))

        if name == "TESTQ-R64-R64":
            logger.debug(flag_solutions)

        # No solutions available
        if not flag_solutions:
            continue

        sg_node = name_to_idx[name]

        register_solutions = list(sg.iter_solutions(sg_node))
        additions = []

        for sol in flag_solutions:
            this_add = []
            for ni, op in flags_sg.iter_solution(node, sol):
                nn = flags_sg.sg.nodes[ni]["label"]
                if nn not in name_to_idx:
                    logger.warning(f"Missing {nn}")
                    continue
                reg_node = name_to_idx[nn]
                this_add.append(reg_node)
            additions.append(this_add)

        for sol in register_solutions:
            # Iterate over all the edges of this solution first
            this_solution = list(sg.iter_solution(sg_node, sol))

            for idx, this_flag_solution in enumerate(additions):
                solid = sol
                edge_idx = len(this_solution)
                if idx > 0:
                    # Need to create a new solution id and copy over all the nodes in the
                    # original solution to this new solution before adding in the flag
                    # solutions.
                    solid = str(uuid.uuid4().hex)[:4]
                    for sidx, item in enumerate(this_solution):
                        nn, data = item
                        sg.sg.add_edge(
                            sg_node,
                            nn,
                            tag=solid,
                            lineno=sidx,
                            ops=data,
                            label=f"({solid}, {sidx}, {data})",
                        )

                for nn in this_flag_solution:
                    if name == "TESTQ-R64-R64":
                        logger.debug(
                            f"{solid} {edge_idx} {sg.sg.nodes[sg_node]['label']}"
                        )
                        logger.debug(f"{sg.sg.nodes[nn]['label']}")

                    sg.sg.add_edge(
                        sg_node,
                        nn,
                        tag=solid,
                        lineno=edge_idx,
                        ops="TODO",
                        label="({solid}, {edge_idx}, TODO)",
                    )
                    edge_idx += 1

def append_equivalences(sg, result_files, outfile):
    # Assumption: The log file names are in the same path and named the same as result
    # files, except, ends with a .log instead of a .yaml.
    nodemap = dict()
    for node in sg.sg.nodes():
        name = sg.sg.nodes[node]["label"]
        nodemap[name] = node

    for file in result_files:
        equivalences = dict()
        task_sfx_to_name = dict()
        name = file[:-4] + "log"

        with open(name) as fd:
            logdata = fd.read().split("\n")

        for line in logdata:
            matches = re.search(r"Added task ([0-9a-f]+): (pseudo-.+)", line)
            if matches:
                sfx = matches.group(1)
                name = matches.group(2).upper()
                task_sfx_to_name[sfx] = name
                continue

            matches = re.search(
                r"Found duplicate ([0-9a-f]+) (pseudo-\S+) ([0-9a-f]+)", line
            )
            if matches:
                name = matches.group(2).upper()
                hash = matches.group(3)
                if hash in task_sfx_to_name:
                    eqname = task_sfx_to_name[hash]
                    equivalences[name] = eqname
                    logger.info(f"Added equiv: {name} -> {eqname}")
                else:
                    logger.warning(f"Unknown suffix: {hash}?")

        for name, node in nodemap.items():
            if name in equivalences:
                eqname = equivalences[name]
                eqnode = nodemap[eqname]
                tag = str(uuid.uuid4().hex)[:4]
                if node == eqnode:
                    logger.warning(f"STOPPED ADDING A SELF EDGE: {node}")
                    continue

                sg.sg.add_edge(
                    node, eqnode, tag=tag, lineno=0, ops="", label=f'({tag}, 0, "")'
                )
                logger.info(f"Added edge {name} ({node}) -> {eqname} ({eqnode}) {tag}")

    # Cleanup unused pseudo-instructions with no outgoing edges
    prune_solutions(sg)

    sg.to_file(outfile + ".pkl")
    logger.success("Generated .pkl file!")

def prune_solutions(sg):
    wl = list(sg.sg.nodes())
    removed = list()
    remove_nodes = list()

    while wl:
        to_remove = list()
        node = wl.pop(0)

        solutions = sg.iter_solutions(node)
        label = sg.sg.nodes[node]["label"]

        if solutions:
            continue

        if not label.startswith("PSEUDO"):
            continue

        for user in list(sg.iter_users(node)):
            data = sg.sg.get_edge_data(user, node)
            for _, item in data.items():
                stag = item["tag"]
                if stag not in removed:
                    removed.append(stag)
                    to_remove.append((user, stag))

        if node not in remove_nodes:
            remove_nodes.append(node)

        for user, stag in to_remove:
            sg.discard_solution(user, stag)
            if user not in wl:
                wl.append(user)

    for node in remove_nodes:
        sg.sg.remove_node(node)

def get_sccs(sg):
    sccs = nx.strongly_connected_components(sg)
    return [scc for scc in sccs if len(scc) > 1]

def analyze_scc(sg, scc):
    loop_solutions = set()
    # Get solution(s) that cause the loop
    for node in scc:
        solutions = sg.iter_solutions(node)
        for solution in solutions:
            for ni, _ in sg.iter_solution(node, solution):
                if ni in scc:
                    loop_solutions.add((node, solution))
                    break

    loop_solutions = list(loop_solutions)
    for node, solution in loop_solutions:
        sg.discard_solution(node, solution)

    # Any remaining solutions for the nodes in the SCC must purely be composed of nodes not a part of this SCC
    non_loop_solutions = []
    for node in scc:
        for solution in sg.iter_solutions(node):
            for lineno, val in enumerate(sg.iter_solution(node, solution)):
                ni, operands = val
                non_loop_solutions.append((node, ni, solution, lineno, operands))

    # Create a "merged" node
    merged_name = " v ".join([sg.sg.nodes[nidx]["label"] for nidx in scc])
    merged_name = f"({merged_name})"
    merged_idx = sg.add_node(merged_name)

    # Redirect all users to the merged node
    for node in scc:
        for user in sg.iter_users(node):
            data = sg.sg.get_edge_data(user, node)
            for _, item in data.items():
                sg.sg.add_edge(user, merged_idx, **item)
            sg.sg.remove_edge(user, node)

    # Add edges for all non-loop solutions
    for sol in non_loop_solutions:
        sg.sg.add_edge(merged_idx, sol[1], tag=sol[2], lineno=sol[3], ops=sol[4])

    # Cleanup and remove the nodes in scc
    for node in scc:
        sg.sg.remove_node(node)

def get_leaves(sg):
    leaves = []
    for node in sg.sg.nodes():
        solutions = sg.iter_solutions(node)
        if not solutions:
            leaves.append(node)

    return [sg.sg.nodes[node]["label"] for node in leaves]

def iter_safe_sets(sg):
    leaves = get_leaves(sg)
    yield leaves

def convert_to_dag(sg):
    sccs = get_sccs(sg.sg)
    for scc in sccs:
        analyze_scc(sg, scc)

def get_program_length_distribution(solutions, safe_set, metric=np.min):
    prog_lengths = defaultdict(lambda: 0)
    program_paths = defaultdict(list)
    worklist = []

    for node in list(reversed(list(nx.topological_sort(solutions.sg)))):
        inst = solutions.sg.nodes[node]["label"]
        if inst in safe_set:
            prog_lengths[inst] = 1
            program_paths[inst] = set([inst])
        else:
            worklist.append(node)

    for node in worklist:
        process_node(solutions, node, prog_lengths, program_paths, metric)

    # Filter out pseudo instructions
    for key in list(prog_lengths.keys()):
        if key.startswith("PSEUDO-"):
            del prog_lengths[key]

    return prog_lengths, program_paths

def process_node(solutions, node, prog_lengths, program_paths, metric):
    # No-op, already processed
    cur_inst = solutions.sg.nodes[node]["label"]
    if cur_inst in prog_lengths:
        return

    current_distribution = []
    inst_distributions = []

    for sol in solutions.iter_solutions(node):
        length_distributions = [0]
        inst_distributions.append(set())

        for nn, op in solutions.iter_solution(node, sol):
            process_node(solutions, nn, prog_lengths, program_paths, metric)
            nn_inst = solutions.sg.nodes[nn]["label"]
            length_distributions.append(prog_lengths[nn_inst])

            inst_distributions[-1].update(program_paths[nn_inst])

        length_distributions = np.sum(length_distributions)
        current_distribution.append(length_distributions)

    if current_distribution:
        value = metric(current_distribution)
        prog_lengths[cur_inst] = value

        # Keep track to be able to reconstruct the solution
        solidx = current_distribution.index(value)
        sol = inst_distributions[solidx]
        program_paths[cur_inst] = sol

def analyze_synthesis_graph(sg, outfile):
    convert_to_dag(sg)

    while True:
        try:
            cycle = nx.find_cycle(sg.sg)
            for item in cycle:
                sg.sg.remove_edge(item[0], item[1])
                logger.warning(f"Removed edge: {item[0]}, {item[1]}")
        except:
            break

    safe_set = list(iter_safe_sets(sg))[0]
    safe_set = [x for x in safe_set]

    logger.info(f"safe set: {sorted(safe_set)}")

    name_to_idx = dict()

    for node in sg.sg.nodes():
        name = sg.sg.nodes[node]["label"]
        name_to_idx[name] = node

    length_distribution_min, pp = get_program_length_distribution(
        sg, safe_set, metric=np.min
    )
    length_distribution_max, _ = get_program_length_distribution(
        sg, safe_set, metric=np.max
    )
    table = dict()
    fan_in = defaultdict(lambda: 0)

    for inst, val in sorted(
        length_distribution_min.items(), key=lambda x: (x[1], x[0])
    ):
        if inst in safe_set:
            continue
        table[inst] = {
            "min": val,
            "max": length_distribution_max[inst],
            "inst_used": pp[inst],
        }
        for sinst in pp[inst]:
            fan_in[sinst] += 1

    choices = [x for x in safe_set if " V " in x]

    logger.debug(f"Safe set size: {len(safe_set)}")
    logger.debug(f"Safe set choices: {len(choices)}")

    choices = [len(x.split(" V ")) for x in choices]

    logger.debug(f"# of Insts in choices: {np.sum(choices)}")
    logger.debug(f"Median of Insts in choices: {np.median(choices)}")
    logger.debug(f"Total number of sets: {np.prod(choices)}")

    results = dict()
    results["min_distribution"] = length_distribution_min
    results["max_distribution"] = length_distribution_max
    results["safe_set"] = safe_set
    results["fan_in"] = fan_in
    results["inst_distribution"] = pp

    return results

if __name__ == "__main__":
    argp = argparse.ArgumentParser()

    argp.add_argument("--pipe", type=str, help="Read results from a streaming pipe")

    argp.add_argument("-o", type=str, help="Write output to filename")

    argp.add_argument(
        "--result-file", nargs="+", type=str, help="Read results from result file"
    )

    argp.add_argument(
        "--continue-partial",
        type=str,
        help="Reload and continue synthesis from a previous synthesis graph",
    )

    argp.add_argument(
        "--flags", type=str, help="File containing syntheis results for flags"
    )

    args = argp.parse_args()

    assert args.pipe or args.result_file

    results = []
    wq = ListWQ([])

    if args.pipe:
        with open(args.pipe) as fd:
            conn = yaml.load(fd)

        # XXX
        conn["wq"] = "synth-results"
        results = BeanstalkWQ(conn)

        conn["wq"] = "synth-wq"
        wq = BeanstalkWQ(conn)
    else:
        jresults = []
        for file in args.result_file:
            with open(file) as fd:
                results = yaml.load(fd)
            for name, res in results.items():
                jresults.append(res.__dict__)
        results = jresults

    with open(
        "cluster-results/synth-results-apr-24-2021/inst-parse-stats-ea39.yaml"
    ) as fd:
        inst_parse_stats = yaml.load(fd)
        successes = inst_parse_stats["success"]

    all_inst_count = 0
    for k, v in inst_parse_stats.items():
        all_inst_count += len(v)

    success_names = [
        x.rsplit("/", 1)[1][:-2].upper().replace("_", "-") for x in successes
    ]

    sg = generate(results, wq, args.o, args.continue_partial)
    append_equivalences(sg, args.result_file, args.o)

    sg.get_safe_sets_brute()

    # Print some basic stats
    instructions = []
    have_solutions = []
    for node in sg.sg.nodes():
        name = sg.sg.nodes[node]["label"]
        if name.startswith("PSEUDO"):
            continue
        instructions.append(name)
        solutions = list(sg.iter_solutions(node))
        if solutions:
            have_solutions.append(name)

    logger.debug(f"All X86 Instructions: {all_inst_count}")
    logger.debug(f"Instructions: {len(instructions)}")
    logger.debug(f"Solutions: {len(have_solutions)}")
    logger.debug(f"Solutions: {set(success_names).difference(set(instructions))}")

    print(
        " ".join([f"'{x}'" for x in set(success_names).difference(set(instructions))])
    )

    if args.flags:
        stitch_flag_solutions(sg, args.flags)
        sg.to_file(args.o + ".pkl")
        logger.success(f"Generated {args.o}.pkl file!")

    analyze_synthesis_graph(sg, args.o)
    sg.to_file(args.o + ".pkl")
    logger.success(f"Generated {args.o}.pkl file!")

{'symbols': 'sympy.symbols', 'reduce': 'functools.reduce', 'operator': 'operator', 'simplify_logic': 'sympy.logic.simplify_logic', 'to_cnf': 'sympy.logic.boolalg.to_cnf', 'to_dnf': 'sympy.logic.boolalg.to_dnf', 'Var': 'nnf.Var', 'Or': 'nnf.Or', 'And': 'nnf.And'}

{'symbols': 'sympy.symbols', 'reduce': 'functools.reduce', 'operator': 'operator', 'simplify_logic': 'sympy.logic.simplify_logic', 'to_cnf': 'sympy.logic.boolalg.to_cnf', 'to_dnf': 'sympy.logic.boolalg.to_dnf', 'Var': 'nnf.Var', 'Or': 'nnf.Or', 'And': 'nnf.And', 'plt': 'matplotlib.pyplot', 'graphviz_layout': 'networkx.drawing.nx_agraph.graphviz_layout', 'write_dot': 'networkx.drawing.nx_agraph.write_dot'}


import unittest
import logging
import numpy as np
from h5py import special_dtype
from h5py import check_dtype
from h5py import Reference
from h5py import RegionReference
from h5json import hdf5dtype

class Hdf5dtypeTest(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        super(Hdf5dtypeTest, self).__init__(*args, **kwargs)
        # main
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.INFO)

    def testBaseIntegerTypeItem(self):
        dt = np.dtype("<i1")
        typeItem = hdf5dtype.getTypeItem(dt)
        self.assertEqual(typeItem["class"], "H5T_INTEGER")
        self.assertEqual(typeItem["base"], "H5T_STD_I8LE")
        typeItem = hdf5dtype.getTypeResponse(typeItem)  # non-verbose format
        self.assertEqual(typeItem["class"], "H5T_INTEGER")
        self.assertEqual(typeItem["base"], "H5T_STD_I8LE")

    def testBaseFloatTypeItem(self):
        dt = np.dtype("<f8")
        typeItem = hdf5dtype.getTypeItem(dt)
        self.assertEqual(typeItem["class"], "H5T_FLOAT")
        self.assertEqual(typeItem["base"], "H5T_IEEE_F64LE")
        typeItem = hdf5dtype.getTypeResponse(typeItem)  # non-verbose format
        self.assertEqual(typeItem["class"], "H5T_FLOAT")
        self.assertEqual(typeItem["base"], "H5T_IEEE_F64LE")

    def testBaseStringTypeItem(self):
        dt = np.dtype("S3")
        typeItem = hdf5dtype.getTypeItem(dt)
        self.assertEqual(typeItem["class"], "H5T_STRING")
        self.assertEqual(typeItem["length"], 3)
        self.assertEqual(typeItem["strPad"], "H5T_STR_NULLPAD")
        self.assertEqual(typeItem["charSet"], "H5T_CSET_ASCII")

    def testBaseStringUTFTypeItem(self):
        dt = np.dtype("U3")
        try:
            # typeItem = hdf5dtype.getTypeItem(dt)
            hdf5dtype.getTypeItem(dt)
            self.assertTrue(False)  # expected exception
        except TypeError:
            pass  # expected

    def testBaseVLenAsciiTypeItem(self):
        dt = special_dtype(vlen=bytes)
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_STRING")
        self.assertEqual(typeItem["length"], "H5T_VARIABLE")
        self.assertEqual(typeItem["strPad"], "H5T_STR_NULLTERM")
        self.assertEqual(typeItem["charSet"], "H5T_CSET_ASCII")
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testBaseVLenUnicodeTypeItem(self):
        dt = special_dtype(vlen=str)
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_STRING")
        self.assertEqual(typeItem["length"], "H5T_VARIABLE")
        self.assertEqual(typeItem["strPad"], "H5T_STR_NULLTERM")
        self.assertEqual(typeItem["charSet"], "H5T_CSET_UTF8")
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testBaseEnumTypeItem(self):
        mapping = {"RED": 0, "GREEN": 1, "BLUE": 2}
        dt = special_dtype(enum=(np.int8, mapping))
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_ENUM")
        baseItem = typeItem["base"]
        self.assertEqual(baseItem["class"], "H5T_INTEGER")
        self.assertEqual(baseItem["base"], "H5T_STD_I8LE")
        self.assertTrue("members" in typeItem)
        mapp_out = dict((m["name"], m["value"]) for m in typeItem["members"])
        self.assertEqual(mapp_out["GREEN"], 1)
        self.assertEqual(typeSize, 1)

    def testBaseBoolTypeItem(self):
        typeItem = hdf5dtype.getTypeItem(np.dtype("bool"))
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_ENUM")
        baseItem = typeItem["base"]
        self.assertEqual(baseItem["class"], "H5T_INTEGER")
        self.assertEqual(baseItem["base"], "H5T_STD_I8LE")
        self.assertTrue("members" in typeItem)
        members = typeItem["members"]
        self.assertEqual(len(members), 2)
        mapping = dict((m["name"], m["value"]) for m in members)
        self.assertEqual(mapping["FALSE"], 0)
        self.assertEqual(mapping["TRUE"], 1)
        self.assertEqual(typeSize, 1)

    def testBaseArrayTypeItem(self):
        dt = np.dtype("(2,2)<int32")
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_ARRAY")
        baseItem = typeItem["base"]
        self.assertEqual(baseItem["class"], "H5T_INTEGER")
        self.assertEqual(baseItem["base"], "H5T_STD_I32LE")
        self.assertEqual(typeSize, 16)

    def testObjReferenceTypeItem(self):
        # dt = np.dtype('S48', metadata={'ref': val.__class__})
        dt = special_dtype(ref=Reference)
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_REFERENCE")
        self.assertEqual(typeItem["base"], "H5T_STD_REF_OBJ")
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testRegionReferenceTypeItem(self):
        dt = special_dtype(ref=RegionReference)
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_REFERENCE")
        self.assertEqual(typeItem["base"], "H5T_STD_REF_DSETREG")
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testCompoundArrayTypeItem(self):
        dt = np.dtype([("a", "<i1"), ("b", "S1", (10,))])
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_COMPOUND")
        fields = typeItem["fields"]
        field_a = fields[0]
        self.assertEqual(field_a["name"], "a")
        field_a_type = field_a["type"]
        self.assertEqual(field_a_type["class"], "H5T_INTEGER")
        self.assertEqual(field_a_type["base"], "H5T_STD_I8LE")
        field_b = fields[1]
        self.assertEqual(field_b["name"], "b")
        field_b_type = field_b["type"]
        self.assertEqual(field_b_type["class"], "H5T_ARRAY")
        self.assertEqual(field_b_type["dims"], (10,))
        field_b_basetype = field_b_type["base"]
        self.assertEqual(field_b_basetype["class"], "H5T_STRING")
        self.assertEqual(typeSize, 11)

    def testCompoundArrayVlenIntTypeItem(self):
        dt_vlen = special_dtype(vlen=np.int32)
        dt_arr = np.dtype((dt_vlen, (4,)))
        dt_compound = np.dtype(
            [("VALUE1", np.float64), ("VALUE2", np.int64), ("VALUE3", dt_arr)]
        )
        typeItem = hdf5dtype.getTypeItem(dt_compound)

        # typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_COMPOUND")
        fields = typeItem["fields"]
        field_a = fields[0]
        self.assertEqual(field_a["name"], "VALUE1")
        field_a_type = field_a["type"]
        self.assertEqual(field_a_type["class"], "H5T_FLOAT")
        self.assertEqual(field_a_type["base"], "H5T_IEEE_F64LE")
        field_b = fields[1]
        self.assertEqual(field_b["name"], "VALUE2")
        field_b_type = field_b["type"]
        self.assertEqual(field_b_type["class"], "H5T_INTEGER")
        self.assertEqual(field_b_type["base"], "H5T_STD_I64LE")
        field_c = fields[2]
        field_c_type = field_c["type"]
        self.assertEqual(field_c_type["class"], "H5T_ARRAY")
        self.assertEqual(field_c_type["dims"], (4,))
        field_c_base_type = field_c_type["base"]
        self.assertEqual(field_c_base_type["class"], "H5T_VLEN")
        self.assertEqual(field_c_base_type["size"], "H5T_VARIABLE")
        field_c_base_base_type = field_c_base_type["base"]
        self.assertEqual(field_c_base_base_type["class"], "H5T_INTEGER")
        self.assertEqual(field_c_base_base_type["base"], "H5T_STD_I32LE")

    def testCompoundArrayVlenStringTypeItem(self):
        dt_vlen = special_dtype(vlen=bytes)
        dt_arr = np.dtype((dt_vlen, (4,)))
        dt_compound = np.dtype(
            [("VALUE1", np.float64), ("VALUE2", np.int64), ("VALUE3", dt_arr)]
        )
        typeItem = hdf5dtype.getTypeItem(dt_compound)

        # typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_COMPOUND")
        fields = typeItem["fields"]
        field_a = fields[0]
        self.assertEqual(field_a["name"], "VALUE1")
        field_a_type = field_a["type"]
        self.assertEqual(field_a_type["class"], "H5T_FLOAT")
        self.assertEqual(field_a_type["base"], "H5T_IEEE_F64LE")
        field_b = fields[1]
        self.assertEqual(field_b["name"], "VALUE2")
        field_b_type = field_b["type"]
        self.assertEqual(field_b_type["class"], "H5T_INTEGER")
        self.assertEqual(field_b_type["base"], "H5T_STD_I64LE")
        field_c = fields[2]
        field_c_type = field_c["type"]

        self.assertEqual(field_c_type["class"], "H5T_ARRAY")
        self.assertEqual(field_c_type["dims"], (4,))
        field_c_base_type = field_c_type["base"]
        self.assertEqual(field_c_base_type["class"], "H5T_STRING")
        self.assertEqual(field_c_base_type["length"], "H5T_VARIABLE")
        self.assertEqual(field_c_base_type["charSet"], "H5T_CSET_ASCII")

    def testOpaqueTypeItem(self):
        dt = np.dtype("V200")
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_OPAQUE")
        self.assertTrue("base" not in typeItem)
        self.assertEqual(typeSize, 200)

    def testVlenDataItem(self):
        dt = special_dtype(vlen=np.dtype("int32"))
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_VLEN")
        self.assertEqual(typeItem["size"], "H5T_VARIABLE")
        baseItem = typeItem["base"]
        self.assertEqual(baseItem["base"], "H5T_STD_I32LE")
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testVlenReferenceDataItem(self):
        ref_dt = special_dtype(ref=Reference)
        dt = special_dtype(vlen=ref_dt)
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_VLEN")
        self.assertEqual(typeItem["size"], "H5T_VARIABLE")
        baseItem = typeItem["base"]
        self.assertEqual(baseItem["base"], "H5T_STD_REF_OBJ")
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testCompoundTypeItem(self):
        dt = np.dtype(
            [("temp", np.float32), ("pressure", np.float32), ("wind", np.int16)]
        )
        typeItem = hdf5dtype.getTypeItem(dt)
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_COMPOUND")
        self.assertTrue("fields" in typeItem)
        fields = typeItem["fields"]
        self.assertEqual(len(fields), 3)
        tempField = fields[0]
        self.assertEqual(tempField["name"], "temp")
        self.assertTrue("type" in tempField)
        tempFieldType = tempField["type"]
        self.assertEqual(tempFieldType["class"], "H5T_FLOAT")
        self.assertEqual(tempFieldType["base"], "H5T_IEEE_F32LE")
        self.assertEqual(typeSize, 10)

        typeItem = hdf5dtype.getTypeResponse(typeItem)  # non-verbose format
        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeItem["class"], "H5T_COMPOUND")
        self.assertTrue("fields" in typeItem)
        fields = typeItem["fields"]
        self.assertEqual(len(fields), 3)
        tempField = fields[0]
        self.assertEqual(tempField["name"], "temp")
        self.assertTrue("type" in tempField)
        tempFieldType = tempField["type"]
        self.assertEqual(tempFieldType["class"], "H5T_FLOAT")
        self.assertEqual(tempFieldType["base"], "H5T_IEEE_F32LE")
        self.assertEqual(typeSize, 10)

    def testCompoundofCompoundTypeItem(self):
        dt1 = np.dtype([("x", np.float32), ("y", np.float32)])
        dt2 = np.dtype([("a", np.float32), ("b", np.float32), ("c", np.float32)])
        dt = np.dtype([("field1", dt1), ("field2", dt2)])
        typeItem = hdf5dtype.getTypeItem(dt)

        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeSize, 20)
        self.assertEqual(typeItem["class"], "H5T_COMPOUND")
        self.assertTrue("fields" in typeItem)
        fields = typeItem["fields"]
        self.assertEqual(len(fields), 2)
        field1 = fields[0]

        self.assertEqual(field1["name"], "field1")
        field1_type = field1["type"]
        self.assertEqual(field1_type["class"], "H5T_COMPOUND")
        field2 = fields[1]

        self.assertEqual(field2["name"], "field2")
        field2_type = field2["type"]
        self.assertEqual(field2_type["class"], "H5T_COMPOUND")

    def testCreateBaseType(self):
        dt = hdf5dtype.createDataType("H5T_STD_U32BE")
        self.assertEqual(dt.name, "uint32")
        self.assertEqual(dt.byteorder, ">")
        self.assertEqual(dt.kind, "u")

        dt = hdf5dtype.createDataType("H5T_STD_I16LE")
        self.assertEqual(dt.name, "int16")
        self.assertEqual(dt.kind, "i")

        dt = hdf5dtype.createDataType("H5T_IEEE_F64LE")
        self.assertEqual(dt.name, "float64")
        self.assertEqual(dt.kind, "f")

        dt = hdf5dtype.createDataType("H5T_IEEE_F32LE")
        self.assertEqual(dt.name, "float32")
        self.assertEqual(dt.kind, "f")

        typeItem = {"class": "H5T_INTEGER", "base": "H5T_STD_I32BE"}
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "int32")
        self.assertEqual(dt.kind, "i")
        self.assertEqual(typeSize, 4)

    def testCreateBaseStringType(self):
        typeItem = {"class": "H5T_STRING", "charSet": "H5T_CSET_ASCII", "length": 6}
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "bytes48")
        self.assertEqual(dt.kind, "S")
        self.assertEqual(typeSize, 6)

    def testCreateBaseUnicodeType(self):
        typeItem = {"class": "H5T_STRING", "charSet": "H5T_CSET_UTF8", "length": 32}
        try:
            # dt = hdf5dtype.createDataType(typeItem)
            hdf5dtype.createDataType(typeItem)
            self.assertTrue(False)  # expected exception
        except TypeError:
            pass

    def testCreateNullTermStringType(self):
        typeItem = {
            "class": "H5T_STRING",
            "charSet": "H5T_CSET_ASCII",
            "length": 6,
            "strPad": "H5T_STR_NULLTERM",
        }
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "bytes48")
        self.assertEqual(dt.kind, "S")
        self.assertEqual(typeSize, 6)

    def testCreateVLenStringType(self):
        typeItem = {
            "class": "H5T_STRING",
            "charSet": "H5T_CSET_ASCII",
            "length": "H5T_VARIABLE",
        }
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "object")
        self.assertEqual(dt.kind, "O")
        self.assertEqual(check_dtype(vlen=dt), bytes)
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testCreateVLenUTF8Type(self):
        typeItem = {
            "class": "H5T_STRING",
            "charSet": "H5T_CSET_UTF8",
            "length": "H5T_VARIABLE",
        }
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "object")
        self.assertEqual(dt.kind, "O")
        self.assertEqual(check_dtype(vlen=dt), str)
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testCreateVLenDataType(self):
        typeItem = {"class": "H5T_VLEN", "base": "H5T_STD_I32BE"}
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "object")
        self.assertEqual(dt.kind, "O")
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testCreateOpaqueType(self):
        typeItem = {"class": "H5T_OPAQUE", "size": 200}
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "void1600")
        self.assertEqual(dt.kind, "V")
        self.assertEqual(typeSize, 200)

    def testCreateEnumType(self):
        typeItem = {
            "class": "H5T_ENUM",
            "base": {"base": "H5T_STD_I16LE", "class": "H5T_INTEGER"},
            "members": [
                {"name": "GAS", "value": 2},
                {"name": "LIQUID", "value": 1},
                {"name": "PLASMA", "value": 3},
                {"name": "SOLID", "value": 0},
            ],
        }

        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeSize, 2)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "int16")
        self.assertEqual(dt.kind, "i")
        mapping = check_dtype(enum=dt)
        self.assertTrue(isinstance(mapping, dict))
        self.assertEqual(mapping["SOLID"], 0)
        self.assertEqual(mapping["LIQUID"], 1)
        self.assertEqual(mapping["GAS"], 2)
        self.assertEqual(mapping["PLASMA"], 3)

    def testCreateBoolType(self):
        typeItem = {
            "class": "H5T_ENUM",
            "base": {"base": "H5T_STD_I8LE", "class": "H5T_INTEGER"},
            "members": [{"name": "TRUE", "value": 1}, {"name": "FALSE", "value": 0}],
        }

        typeSize = hdf5dtype.getItemSize(typeItem)
        self.assertEqual(typeSize, 1)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "bool")
        self.assertEqual(dt.kind, "b")

    def testCreateCompoundType(self):
        typeItem = {
            "class": "H5T_COMPOUND",
            "fields": [
                {"name": "temp", "type": "H5T_IEEE_F32LE"},
                {"name": "pressure", "type": "H5T_IEEE_F32LE"},
                {
                    "name": "location",
                    "type": {
                        "length": "H5T_VARIABLE",
                        "charSet": "H5T_CSET_ASCII",
                        "class": "H5T_STRING",
                        "strPad": "H5T_STR_NULLTERM",
                    },
                },
                {"name": "wind", "type": "H5T_STD_I16LE"},
            ],
        }
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "void144")
        self.assertEqual(dt.kind, "V")
        self.assertEqual(len(dt.fields), 4)
        dtLocation = dt[2]
        self.assertEqual(dtLocation.name, "object")
        self.assertEqual(dtLocation.kind, "O")
        self.assertEqual(check_dtype(vlen=dtLocation), bytes)
        self.assertEqual(typeSize, "H5T_VARIABLE")

    def testCreateCompoundOfCompoundType(self):
        typeItem = {
            "class": "H5T_COMPOUND",
            "fields": [
                {
                    "name": "field1",
                    "type": {
                        "class": "H5T_COMPOUND",
                        "fields": [
                            {
                                "name": "x",
                                "type": {
                                    "class": "H5T_FLOAT",
                                    "base": "H5T_IEEE_F32LE",
                                },
                            },
                            {
                                "name": "y",
                                "type": {
                                    "class": "H5T_FLOAT",
                                    "base": "H5T_IEEE_F32LE",
                                },
                            },
                        ],
                    },
                },
                {
                    "name": "field2",
                    "type": {
                        "class": "H5T_COMPOUND",
                        "fields": [
                            {
                                "name": "a",
                                "type": {
                                    "class": "H5T_FLOAT",
                                    "base": "H5T_IEEE_F32LE",
                                },
                            },
                            {
                                "name": "b",
                                "type": {
                                    "class": "H5T_FLOAT",
                                    "base": "H5T_IEEE_F32LE",
                                },
                            },
                            {
                                "name": "c",
                                "type": {
                                    "class": "H5T_FLOAT",
                                    "base": "H5T_IEEE_F32LE",
                                },
                            },
                        ],
                    },
                },
            ],
        }
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "void160")
        self.assertEqual(dt.kind, "V")
        self.assertEqual(len(dt.fields), 2)
        dt_field1 = dt[0]
        self.assertEqual(dt_field1.name, "void64")
        self.assertEqual(dt_field1.kind, "V")
        self.assertEqual(len(dt_field1.fields), 2)
        dt_field2 = dt[1]
        self.assertEqual(dt_field2.name, "void96")
        self.assertEqual(dt_field2.kind, "V")
        self.assertEqual(len(dt_field2.fields), 3)

    def testCreateCompoundTypeUnicodeFields(self):
        typeItem = {
            "class": "H5T_COMPOUND",
            "fields": [
                {"name": u"temp", "type": "H5T_IEEE_F32LE"},
                {"name": u"pressure", "type": "H5T_IEEE_F32LE"},
                {"name": u"wind", "type": "H5T_STD_I16LE"},
            ],
        }
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "void80")
        self.assertEqual(dt.kind, "V")
        self.assertEqual(len(dt.fields), 3)
        self.assertEqual(typeSize, 10)

    def testCreateArrayType(self):
        typeItem = {"class": "H5T_ARRAY", "base": "H5T_STD_I64LE", "dims": (3, 5)}
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "void960")
        self.assertEqual(dt.kind, "V")
        self.assertEqual(typeSize, 120)

    def testCreateArrayIntegerType(self):
        typeItem = {"class": "H5T_INTEGER", "base": "H5T_STD_I64LE", "dims": (3, 5)}

        try:
            hdf5dtype.createDataType(typeItem)
            self.assertTrue(
                False
            )  # expected exception - dims used with none array type
        except TypeError:
            pass  # should get exception

    def testCreateVlenObjRefType(self):
        typeItem = {
            "class": "H5T_VLEN",
            "base": {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_OBJ"},
        }
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(dt.name, "object")
        self.assertEqual(dt.kind, "O")
        self.assertTrue(check_dtype(ref=dt) is None)
        dt_base = check_dtype(vlen=dt)
        self.assertTrue(dt_base is not None)
        self.assertTrue(check_dtype(ref=dt_base) is Reference)

    def testCreateCompoundArrayType(self):
        typeItem = {
            "class": "H5T_COMPOUND",
            "fields": [
                {"type": {"base": "H5T_STD_I8LE", "class": "H5T_INTEGER"}, "name": "a"},
                {
                    "type": {
                        "dims": [10],
                        "base": {
                            "length": 1,
                            "charSet": "H5T_CSET_ASCII",
                            "class": "H5T_STRING",
                            "strPad": "H5T_STR_NULLPAD",
                        },
                        "class": "H5T_ARRAY",
                    },
                    "name": "b",
                },
            ],
        }
        typeSize = hdf5dtype.getItemSize(typeItem)
        dt = hdf5dtype.createDataType(typeItem)
        self.assertEqual(len(dt.fields), 2)
        self.assertTrue("a" in dt.fields.keys())
        self.assertTrue("b" in dt.fields.keys())
        self.assertEqual(typeSize, 11)

if __name__ == "__main__":
    # setup test files

    unittest.main()


#!/usr/bin/python2.7
# -*- coding: latin-1 -*-
import keras.optimizers as optimizers
import numpy as np
import pandas as pd

try:
    reload  # Python 2.7
except NameError:
    try:
        from importlib import reload  # Python 3.4+
    except ImportError:
        from imp import reload  # Python 3.0 - 3.3

import matplotlib as mpl
import matplotlib.pyplot as plt
import sys, warnings, argparse

import os
import sys
import os.path as osp

import time

## Import de la classe CVC ##
cases_folder = osp.abspath(osp.dirname("../cases/"))
sys.path.append(cases_folder)

import tensorflow as tf

from keras.models import Model
from keras.models import Sequential
from keras.layers import Input, Flatten, Dense, MaxPool1D, UpSampling1D, Activation, Conv1D

import Class_Vit_Choc as cvc
cvc = reload(cvc)

from visu_strides import glide1D

try :
    tf.reset_default_graph()
except :
    pass

def model():
    # Unet
    input_shape = (82,1)

    input_data = Input(shape=input_shape)
    x = Conv1D(2, kernel_size=3, padding='same')(input_data)
    #Rajouter une couche BN
    x = Activation('relu')(x)
    print ("in --> Conv1D(32, 2, same) = {}".format(x.get_shape())) # (?, 81, 32)

    x = Conv1D(4, kernel_size=2, padding='same')(input_data)
    x = Activation('relu')(x)
    print ("Conv1D --> Conv1D(32, 2, same). Out_shape = {} \n".format(x.get_shape()))
    #print x.get_shape() # (?, 81, 32)

    in_pool_shape = x.get_shape()

    x = MaxPool1D(pool_size=2, padding='same')(x)
    print ("MaxPool1D pool_size = 2, padding same")
    print ("MaxPool1D(1st enc): in_shape = {}\tOut_shape = {}".format(in_pool_shape, x.get_shape()))
    print (" ")
    #

    x = Conv1D(8, kernel_size=2, padding='same')(x)
    #Rajouter une couche BN
    x = Activation('relu')(x)
    print ("Pooled --> Conv1D(64, 2, same). Out_shape = {}".format(x.get_shape()))

    x = Conv1D(64, kernel_size=2, padding='same')(x)
    #Rajouter une couche BN
    x = Activation('relu')(x)
    print ("Conv1D --> Conv1D(64, 2, same). Out_shape = {} \n".format(x.get_shape()))

    in_pool_shape = x.get_shape()

    x = MaxPool1D(pool_size=2, padding='same')(x)
    print ("MaxPool1D pool_size = 2, padding same")
    print ("MaxPool1D(2nd enc): in_shape = {}\tOut_shape = {}".format(in_pool_shape, x.get_shape()))
    print (" ")

    x = Conv1D(16, kernel_size=2, padding='same')(x)
    #Rajouter une couche BN
    x = Activation('relu')(x)
    print ("Pooled --> Conv1D(128, 2, same). Out_shape = {}".format(x.get_shape()))

    x = Conv1D(16, kernel_size=2, padding='same')(x)
    #Rajouter une couche BN
    x = Activation('relu')(x)
    print ("Conv1D --> Conv1D(128, 2, same). Out_shape = {} \n".format(x.get_shape()))

    in_samp_shape = x.get_shape()

    x = UpSampling1D(2)(x)
    print ("Upsampling pool_size = 2")
    print ("UpSampling(1st dec): in_shape= {}\tOut_shape = {}".format(in_samp_shape, x.get_shape()))
    print (" ")

    x = Conv1D(8, kernel_size=2, padding='same')(x)
    #Rajouter une couche BN
    x = Activation('relu')(x)
    print ("Upsampled --> Conv1D(64, 2, same). Out_shape = {}".format(x.get_shape()))

    x = Conv1D(4, kernel_size=2, padding='same')(x)
    #Rajouter une couche BN
    x = Activation('relu')(x)
    print ("Conv1D --> Conv1D(64, 2, same). Out_shape = {} \n".format(x.get_shape()))

    in_samp_shape = x.get_shape()

    x = UpSampling1D(2)(x)
    print ("Upsampling pool_size = 2")
    print ("UpSampling(2nd dec): in_shape= {}\tOut_shape = {}".format(in_samp_shape, x.get_shape()))
    print (" ")

    x = Conv1D(2, kernel_size=3, padding='valid')(x)
    #Rajouter une couche BN
    x = Activation('relu')(x)
    print ("Upsampled --> Conv1D(32, 2, same). Out_shape = {}".format(x.get_shape()))

    x = Conv1D(1, kernel_size=1, padding='valid')(x)
    #Rajouter une couche BN
    output = Activation('relu')(x)
    print ("Conv1D --> Conv1D(1, 1, same) (Output). Out_shape = {}".format(x.get_shape()))

    unet = Model(input_data, output, name='u_net_out')

    return unet
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# u_it139_0_Nt52_Nx82_CFL0_4_nu0_05_sin.npy

parser = cvc.parser()
cb = cvc.Vitesse_Choc(parser)

def build_dataset(cb) :
    nu52_avail = [0.05, 0.035, 0.025, 0.015]
    Nt = 52
    Nx = 82
    CFL = '0_4'
    str_nu52_avail = [str(i).replace('.', '_') for i in nu52_avail]

#    u_it168_4_Nt52_Nx82_CFL0_4_nu0_05_sin.npy

    true_data =lambda it,neval,s_nu : "u_it%d_%d_Nt52_Nx82_CFL0_4_nu%s_sin.npy" % (it, neval, s_nu)

    warp_name = lambda it, neval, s_nu : osp.join(cb.datapath, true_data(it, neval, s_nu))

    u_means = dict()

    uu = np.zeros((Nx))
    tempu = np.zeros(Nx)

    path_dataset = osp.join(cb.datapath, 'burger_matrices/cas_u')

    std_name = lambda std, it, s_nu : '%s_Nx_82_Nt_52_nu_%s_typei_sin_CFL_0_4_it_%03d.npy' % (std, s_nu, it)

    key_dict = lambda it, s_nu : "u_it%d_Nt52_Nx_82_nu%s" %(it, s_nu)

    ### Calcul de la moyenne
    for s in str_nu52_avail :
        for i in range(cb.itmax) :
            tempu = np.zeros(Nx)
            for ev in range(cb.num_real) :
                uu = np.load(warp_name(i, ev, s))
                for j in range(0, Nx) :
                    tempu[j-1] += uu[j] / cb.num_real

            u_means[key_dict(i, s)] = tempu

    X, y = np.zeros_like(cb.line_x), np.zeros_like(cb.line_x)
    cnt = 0

    X_keys, y_keys = dict(), dict()

    for s in str_nu52_avail :
        for i in range(cb.itmax) :
            betafile = os.path.join(cb.datapath, "burger_matrices/cas_u/betas", std_name("beta", i, s))
#            chol_file = os.path.join(cb.datapath, "burger_matrices/cas_u/cholesky", std_name(chol, i, s))

            vel_file = u_means[key_dict(i, s)]

            X = np.block([ [X], [vel_file] ])
            y = np.block([ [y], [np.load(betafile)] ])

            X_keys[cnt] = key_dict(i, s)
            y_keys[cnt] = betafile

            cnt += 1

    X = np.delete(X, 0, axis=0)
    y = np.delete(y, 0, axis=0)
    return X, y, (cnt, X_keys, y_keys, u_means)

def split_data(X, y):
    permutation = np.random.permutation(len(y))

    X, y = np.copy(X), np.copy(y)

    X_rand = X[permutation]
    y_rand = y[permutation]

    xtr = X_rand[:int(len(y)*0.8)]
    ytr = y_rand[:int(len(y)*0.8)]

    xte = X_rand[int(len(y)*0.8):int(len(y)*0.9)]
    yte = y_rand[int(len(y)*0.8):int(len(y)*0.9)]

    xval = X_rand[int(len(y)*0.9):]
    yval = y_rand[int(len(y)*0.9):]

    return xtr, ytr, xte, yte, xval, yval

def see_pred(cb, unet, u_means, key, indice) :
    u_n = xval[indice]
    beta_pred = unet.predict(u_n.reshape(1, 82, 1)).ravel()

    u_n = u_n.ravel()
    true_beta = yval[indice].ravel()

#    u_beta(self, beta, u_n, verbose=False)
    u_npred = cb.u_beta(beta_pred, u_n)

#    u_nexact =

if __name__ == '__main__':
    #    run brgrs1D_beta.py -nu 5e-2 -itmax 180 -CFL 0.4 -num_real 5 -Nx 82 -Nt 32 -typeJ "u" -dp "./../cases/data/2019_burger_data/"
    parser = cvc.parser()
    cb = cvc.Vitesse_Choc(parser)

    unet = model()
    X, y, utils = build_dataset(cb)

    u_means = utils[-1]

    xtr, ytr, xte, yte, xval, yval = split_data(X, y)

    from keras.callbacks import TensorBoard

    adam = optimizers.Adam(lr=1e-2, decay=1e-5)
    unet.compile(loss='mean_squared_error', optimizer=adam)

    x_train = np.reshape(xtr, (len(xtr), 82, 1))
    y_train = np.reshape(ytr, (len(ytr), 82, 1))

    x_test = np.reshape(xte, (len(xte), 82, 1))
    y_test = np.reshape(yte, (len(yte), 82, 1))

    x_validation = np.reshape(xval, (len(xval), 82, 1))
    y_validation = np.reshape(yval, (len(yval), 82, 1))

    unet.fit(x_train, y_train,
                epochs=10,
#                batch_size=32,
                shuffle=True,
                validation_data=(x_test, y_test))
#                callbacks=[TensorBoard(log_dir='/tmp/autoencoder')])


import json

def read(file):
    with open(file) as file:
        data = json.load(file)
        city = data["city"]
        bots = data["bots"]
        population = data["population"]

        print(f"\nCity Name: {city}")
        print(f"\nPopulation Size: {population}")
        for bot in bots:
            bot_name = bot["name"]
            bots_stats = bot["stats"]
            bots_stats_speed = bots_stats["speed"]
            bots_stats_strength = bots_stats["strength"]
            print(f"\nBot Name: {bot_name}\nSpeed: {bots_stats_speed}\nStrength: {bots_stats_strength}")

def run():
    read("robocity.json")

if __name__ == "__main__":
    run()


import argparse
import dataloader
from model import IEMSAModel
import torch
import torch.nn as nn
import torch.optim as optim
import datetime
from tensorboardX import SummaryWriter
from recorder import Recorder

def epoch(epoch_idx, is_train):
    model.train() if is_train else model.eval()
    loader = train_loader if is_train else val_loader
    recorder.epoch_start(epoch_idx, is_train, loader)
    for batch_idx, batch in enumerate(loader):
        batch_size = batch['response'].size()[0]
        batch = {key: val.to(device) for key, val in batch.items()}
        optimizer.zero_grad()
        output = model(batch)
        loss = criterion(output[1], batch['response'][:, 1:])
        for logit, batch_key in zip(output[0], ['post_2', 'post_3', 'post_4']):
            loss += criterion(logit, batch[batch_key][:, 1:])
        if is_train:
            loss.backward()
            optimizer.step()
        recorder.batch_end(batch_idx, batch_size, loss.item())
    recorder.log_text(output, batch)
    recorder.epoch_end()

def train():
    for epoch_idx in range(1, args.epochs + 1):
        epoch(epoch_idx, is_train=True)
        epoch(epoch_idx, is_train=False)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='parser')
    parser.add_argument('--project', type=str, default='iemsa')
    parser.add_argument('--timestamp', type=str, default=datetime.datetime.now().strftime("%y%m%d%H%M%S"))
    parser.add_argument('--data_dir', type=str, default='data')
    parser.add_argument('--log_dir', type=str, default='log')
    parser.add_argument('--batch_size', type=int, default=128)
    parser.add_argument('--epochs', type=int, default=100)
    parser.add_argument('--lr', type=float, default=1e-4)
    parser.add_argument('--log_interval', type=int, default=100)
    parser.add_argument('--glove_path', type=str, default='data/glove.6B.200d.txt')
    parser.add_argument('--d_embed', type=int, default=200)
    parser.add_argument('--d_hidden', type=int, default=256)
    parser.add_argument('--d_context', type=int, default=256) # msa context vector
    parser.add_argument('--n_word_vocab', type=int, default=10000)
    parser.add_argument('--n_rel_vocab', type=int, default=45)
    parser.add_argument('--n_layer', type=int, default=2)
    parser.add_argument('--max_decode_len', type=int, default=10)
    parser.add_argument('--seed', type=int, default=42)
    parser.add_argument('--num_workers', type=int, default=4)
    parser.add_argument('--no_cuda', action='store_true')
    args = parser.parse_args()

    device = torch.device("cuda:0" if not args.no_cuda and torch.cuda.is_available() else "cpu")
    torch.manual_seed(args.seed)
    train_loader = dataloader.get_dataloader(args.data_dir, 'train', args.batch_size, args.num_workers)
    val_loader = dataloader.get_dataloader(args.data_dir, 'val', args.batch_size, args.num_workers)
    model = IEMSAModel(args, train_loader.dataset.idx2word)
    model = model.to(device)
    optimizer = optim.Adam(model.parameters(), args.lr)
    criterion = nn.CrossEntropyLoss(ignore_index=0)
    writer = SummaryWriter(f'{args.log_dir}/{args.project}_{args.timestamp}')
    recorder = Recorder(args, writer, train_loader.dataset.idx2word)

    train()


from typing import Optional

import discord
from discord import app_commands

from ns_bot.controllers.base_nationstate_controller import BaseNationstateController
from ns_bot.data.shards import (
    SPECIAL_WORLD_ASSEMBLY_SHARDS,
    VALID_WORLD_ASSEMBLY_SHARDS,
)
from ns_bot.formatting.world_assembly_info import FormatWAInfo

class WorldAssemblyController(BaseNationstateController):
    @classmethod
    async def wa_shards_autocomplete(
        cls, _: discord.Interaction, current: str
    ) -> list[app_commands.Choice[str]]:
        return cls.shard_autocomplete(current, VALID_WORLD_ASSEMBLY_SHARDS)

    async def info(
        self,
        interaction: discord.Interaction,
        council: app_commands.Choice[int],
        shard: Optional[str] = None,
    ):
        shards = [shard] if shard else None
        if shard.lower() in SPECIAL_WORLD_ASSEMBLY_SHARDS:
            shards.append("resolution")
        data = await self.bot.nationstates_api.get_wa_data(council.value, shards=shards)
        await FormatWAInfo.format(council.name, shard, data, interaction)


from PIL import Image, ImageDraw

imagen = Image.open('gatito.jpg')

rojos = [0 for _ in range(256)]
verdes = [0 for _ in range(256)]
azules = [0 for _ in range(256)]

for r, g, b in imagen.getdata():
    rojos[r] += 1
    verdes[g] += 1
    azules[b] += 1

def generar_histograma(conteos, nombre_archivo):
    lienzo = Image.new('RGB', (256, 256))
    lapicero = ImageDraw.Draw(lienzo)

    maximo_conteo = max(conteos)
    for color, conteo in enumerate(conteos):
        factor = conteo / maximo_conteo
        lapicero.line([(color, 256), (color, 256 - (factor * 256))])

    lienzo.save(nombre_archivo)

generar_histograma(rojos, 'rojos.jpg')
generar_histograma(verdes, 'verdes.jpg')
generar_histograma(azules, 'azules  .jpg')


import streamlit as st
from helpers import set_bg_hack_url

def set_background_widget(background_url):
    set_bg_hack_url(background_url)

def sidebar_widget():
    st.sidebar.title("Options")

    input_media = st.sidebar.radio(label="Search with...", options=["text", "image"])

    st.sidebar.markdown(
        """This example lets you use a *textual* description to search through *images* of tattoos using [Jina](https://github.com/jina-ai/jina/).
        """
    )

    st.sidebar.markdown(
        "[Repo link](https://github.com/k-zehnder/jina-clip-streamlit)"
    )

    st.sidebar.markdown(
        "[Credits](https://github.com/jina-ai/example-multimodal-fashion-search)"
    )
    return input_media


from flask import Flask, render_template, request
import chatbot as mymodule

app = Flask(__name__)

# app.static_folder='static'

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/get')
def get_bot_response():
    userAnswer = request.args.get('msg')
    chat = mymodule.ChatBot(userAnswer)
    return str(chat.check_answer())

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)


#!/usr/bin/env python3

import sys
import csv
import json
from termcolor import colored

# First clean csv, we can then re-use the csvToJSON script for all datasets.
source_url = "https://www.oscr.org.uk/about-charities/search-the-register/charity-register-download"
source_date = "2019"

# some of these are aliasses of other names also in the list
# Names from: https://www.geni.com/projects/Counties-of-Scotland-United-Kingdom/14402
counties = [
        "Aberdeenshire",
        "Angus",
        "Argyllshire",
        "Ayrshire",
        "Banffshire",
        "Moray council",
        "Aberdeenshire council",
        "Berwickshire",
        "Buteshire",
        "Caithness",
        "Clackmannanshire",
        "Dumfriesshire",
        "Dumfries",
        "Galloway)",
        "Dunbartonshire",
        "Lothian-Main-Page",
        "East Lothian",
        "Haddingtonshire",
        "Midlothian",
        "Fife/Fifeshire",
        "Inverness-shire",
        "Kincardineshire",
        "Kinross-shire",
        "Perth",
        "Kinross",
        "Kirkcudbrightshire",
        "Lanarkshire",
        "Linlithgowshire",
        "West Lothian",
        "Midlothian",
        "Edinburghshire",
        "Moray",
        "Morayshire",
        "Elginshire",
        "Nairnshire-Main-Page",
        "Nairn",
        "Nairnshire",
        "Highland",
        "Orkney",
        "Peebles-shire",
        "Borders",
        "Perthshire",
        "Perth",
        "Kinross",
        "Stirling",
        "Renfrewshire",
        "Ross and Cromarty",
        "Roxburghshire",
        "Selkirkshire",
        "Shetland",
        "Stirlingshire",
        "Sutherland",
        "West Lothian",
        "Linlithgowshire",
        "Wigtownshire"
    ]

# Categories from the 'purpose' field that have these values
# will be placed in the generic category 'human'.
nature_category_list = [
        "The advancement of environmental protection or improvement",
        ]

animal_category_list = [
        "The advancement of animal welfare"
        ]

education_category_list = [
        "The advancement of education",
        ]

health_category_list = [
        "The advancement of health",
        ]

community_category_list = [
        "The advancement of citizenship or community development",
        "The prevention or relief of poverty",
        "The saving of lives",
        "The provision of recreational facilities, or the organisation of recreational activities, with the object of improving the conditions of life for the persons for whom the facilities or activities are primarily intended",
        "The advancement of human rights, conflict resolution or reconciliation",
        "The promotion of religious or racial harmony",
        "The promotion of equality and diversity",
        "The relief of those in need by reason of age, ill health, disability, financial hardship or other disadvantage",
        ]

religion_category_list = [
        "The advancement of religion",
        ]

culture_category_list = [
        "The advancement of the arts, heritage, culture or science",
        ]

sports_category_list = [
        "The advancement of public participation in sport",
        ]

def in_list(purpose, category_list):
    for cat in category_list:
        if cat in purpose:
            return 1
    return 0

def get_category_values(purposes):
    # Using 0 as false, 1 as true
    # Because more compact in search engine (would otherwise write "True")

    # clean purpose from quotes
    pur = purposes.strip().replace("'", "")

    nature = in_list(pur, nature_category_list)
    animal = in_list(pur, animal_category_list)
    education = in_list(pur, education_category_list)
    health = in_list(pur, health_category_list)
    community = in_list(pur, community_category_list)
    religion = in_list(pur, religion_category_list)
    culture = in_list(pur, culture_category_list)
    sports = in_list(pur, sports_category_list)

    return nature, animal, education, health, community, religion, culture, sports

def get_values_from_address(address):
    '''
    "Sports Pavilion, Pentcaitland, East Lothian"
    "The Old Garage, Mill Hills Farm, Crieff, Perthshire"
    "29A Westburn Drive, Aberdeen"
    "60 Brookfield Place, Alva, Clackmannanshire"
    "21 Church Street, Brechin, Angus"
    "Keppochan Farmhouse, Cladich, Dalmally"
    "Meeks Road Surgery, 10 Meeks Road, Falkirk"
    "24 Lanark Road, Edinburgh"
    "6 annerley court, cuparhead, coatbridge, lanarkshire"
    '''
    city = ""
    state = ""  # County / Shire
    parts = address.split(",")
    found_state = False
    # we 'reverse' parts, because we expect our values
    # to be within last 3 items.
    for part in reversed(parts):
        if not found_state:
            for county in counties:
                if part.strip() in county:
                    state = county.strip()
                    found_state = True
        else:
            city = part.strip()
            return city, state

    # couldn't find county, lets assume city is last
    city = address.split(",")[-1].strip()
    return city, state

def fix_nulls(s):
    for line in s:
        yield line.replace('\0', ' ')

def parse(input_file_path, output_file_path):

    # we assume it data has heading like this:
    """
    Charity Number,
    Charity Name,
    Registered Date,
    Known As,
    Charity Status,
    Notes,
    Postcode,
    Constitutional Form,
    Previous Constitutional Form 1,
    Geographical Spread,
    Main Operating Location,
    Purposes,
    Beneficiaries,
    Activities,
    Objectives,
    Principal Office/Trustees Address,
    Website,
    Most recent year income,
    Most recent year expenditure,
    Mailing cycle,
    Year End,
    Donations and legacies income,
    Charitable activities income,
    Other trading activities income,
    Investments income,
    Other income,
    Raising funds spending,
    Charitable activities spending,
    Other spending,
    Parent charity name,
    Parent charity number,
    Parent charity country of registration,
    Designated religious body,
    Regulatory Type
    """
    # For new file we are going to first rewrite the heading
    # We keep: Legal Name, City, State, Country, Website
    # We add: Source URL, Source date
    end_part = source_url + "," + source_date + "\n"

    print("Opening output file..")
    with open(output_file_path, 'w+', encoding="utf-8") as output_file:

        print("Write header..")
        fieldnames = [
                "OfficialID",
                "Name",
                "City",
                "State",
                "Country",
                "Website",
                "NAT",
                "ANI",
                "EDU",
                "HEA",
                "COM",
                "REL",
                "CUL",
                "SPO",
                "SourceURL",
                "SourceDate"
                ]
        output_writer = csv.DictWriter(output_file,
                                        fieldnames=fieldnames,
                                        delimiter=',',
                                        quotechar='"',
                                        quoting=csv.QUOTE_MINIMAL)
        output_writer.writeheader();

        # open to read source
        print("Opening source file..")

        with open(input_file_path, 'r') as input_file:

            input_reader = csv.DictReader(fix_nulls(input_file))

            for line in input_reader:
                city, state = get_values_from_address(line["Principal Office/Trustees Address"])
                nature, animal, education, health, community, religion, culture, sports = get_category_values(line["Purposes"])

                output_writer.writerow({"OfficialID": line["Charity Number"],
                                        "Name": line["Charity Name"],
                                        "City": city,
                                        "State": state,
                                        "Country": "SC",
                                        "Website": line["Website"],
                                        "NAT": nature,
                                        "ANI": animal,
                                        "EDU": education,
                                        "HEA": health,
                                        "COM": community,
                                        "REL": religion,
                                        "CUL": culture,
                                        "SPO": sports,
                                        "SourceURL": source_url,
                                        "SourceDate": source_date})

    # print first couple of lines as check
    with open(output_file_path) as lines:
        print(colored("First two lines of created file:", 'green') )
        print(lines.readline(), end='')
        print(lines.readline(), end='')

    print(colored("Ended cleaning CSV file", "green"))


# -*- coding: utf-8 -*-

'''Un 'acortador de URLs pero que permite:

* Editar adonde apunta el atajo más tarde
* Definir tests para saber si el atajo es válido

'''

# Usamos storm para almacenar los datos
from storm.locals import *

# Usamos bottle para hacer el sitio
from bottle import route, run, request, response, send_file, abort, view, debug, redirect, static_file

# Caracteres válidos en un atajo de URL
from string import letters, digits
validos = letters + digits

def id_to_slug(n):
    '''Dado un entero n, devuelve un "slug", un string
    con letras y números.

    Básicamente un contador devuelve el número
    codificado en base 62 usando a-zA-Z09 como dígitos, y
    dado vuelta (más significativo a la derecha)

    >>> id_to_slug(100000)
    '4aA'
    >>> id_to_slug(100001)
    '5aA'
    '''

    s = ''
    while n:
        s+=validos[n%62]
        n=n//62
    return s

def slug_to_id(slug):
    '''Dado un slug, devuelve un id entero.

    La recíproca de id_to_slug()

    >>> slug_to_id('4aA')
    100000

    '''

    i=0
    for p,l in enumerate(slug):
        i+=62**p*validos.index(l)
    return i

# Creamos una base SQLite
database = create_database("sqlite:///pyurl.sqlite")
store = Store (database)
try:
    # Creamos la tabla
    store.execute ('''
    CREATE TABLE atajo (
        id INTEGER PRIMARY KEY,
        url VARCHAR,
        test VARCHAR
    )
    '''
    )
except:
    pass

class Atajo(object):
    '''Un atajo de URL, que se almacena en una base
    de datos via storm.

    Miembros:

    id    = Único, creciente, entero (primary key)
    url   = la URL original
    test  = un test de validez de la URL
    '''

    __storm_table__ = "atajo"
    id    = Int(primary=True)
    url   = Unicode()
    test  = Unicode()

    def __init__(self, url):
        '''Exigimos la URL, lo demás es opcional.'''
        self.url = url

#Estático
@route('/css/:filename')
def static_file(filename):
    send_file(filename, root='./static/css/')

# Alta de URL
@route('/')
@view('add')
def alta():
    resp = {
        'baseurl':'http://127.0.0.1:8080/',
        'url':'',
        'short':'',
        }
    if 'url' in request.GET:
        u=request.GET['url']
        resp['url']=u
        atajo=Atajo(unicode(u))
        store.add(atajo)
        store.flush()
        resp['short']=id_to_slug(atajo.id)
        store.commit()
    return resp

#Redirigir
@route('/:slug')
def redir(slug):
    atajo=store.find(Atajo, id=slug_to_id(slug)).one()
    if atajo:
        redirect(atajo.url)
    else:
        redirect('/')

if __name__=='__main__':
    # Correr server de prueba
    debug(True)
    run(reloader=False, host='127.0.0.1', port=8080)


import simplejson as json
from flask import Blueprint, request, jsonify
from sqlalchemy import func
import config

# Import models
from app.models import (
    User,
    Lot,
    Tweet,
    db
)

# Define the blueprint: 'users', set its url prefix: app.url/users
user_mod = Blueprint('users', __name__, url_prefix='/users')

@user_mod.route('/<screen_name>/tweets', methods=['GET'])
def get_user_tweets(screen_name):
    tweets = []
    page = request.args.get('page')
    if page is None:
        page = 1
    else:
        page = int(page)
    per_page = request.args.get('per_page')
    if per_page is None:
        per_page = config.TWEETS_PER_PAGE
    else:
        per_page = int(per_page)
    for tweet_obj in Tweet.query.join(
            'user'
        ).filter(
            func.lower(User.screen_name)==func.lower(screen_name)
        ).paginate(page, per_page, False).items:
        tweets.append(json.loads(tweet_obj.json_str))
    return jsonify(tweets=tweets, message='success')