import json
import random
import numpy as np
import scipy
import seaborn as sns
import networkx as nx
import matplotlib.pyplot as plt
from DPC import DPClustering, pairwise_distance

def shortest_path_len(G, i, j):
    return nx.shortest_path_length(G, i, j)

def add_edges(ax, G, pos, **kwargs):
    nx.draw_networkx_edges(G, pos,
                           alpha=kwargs.get('alpha', 0.1),
                           width=kwargs.get('width', 1),
                           ax=ax[0, 0])
    nx.draw_networkx_edges(G, pos,
                           alpha=kwargs.get('alpha', 0.1),
                           width=kwargs.get('width', 1),
                           ax=ax[1, 0])

def cluster_plot(G, pos, true_tags, pred_tags, node_size=400, edge_width=1, label=True, show=True, desc='', **kwargs):
    # large plots
    fig, ax = plt.subplots(1, 2, figsize=(30, 15))
    nx.draw_networkx_edges(G, pos, alpha=0.2, width=edge_width, ax=ax[0])
    nx.draw_networkx_nodes(G, pos,
                           nodelist=G.nodes,
                           node_color=true_tags,
                           node_size=node_size,
                           cmap=plt.cm.jet,
                           alpha=0.8,
                           ax=ax[0])
    if label:
        nx.draw_networkx_labels(G, pos, dict(zip(G.nodes, G.nodes)), font_color="whitesmoke", ax=ax[0])
    ax[0].set_title('Truth')
    nx.draw_networkx_edges(G, pos, alpha=0.2, width=edge_width, ax=ax[1])
    nx.draw_networkx_nodes(G, pos,
                           nodelist=G.nodes,
                           node_color=pred_tags,
                           node_size=node_size,
                           cmap=plt.cm.jet,
                           alpha=0.8,
                           ax=ax[1])
    if label:
        nx.draw_networkx_labels(G, pos, dict(zip(G.nodes, G.nodes)), font_color="whitesmoke", ax=ax[1])
    ax[1].set_title(f'Prediction {desc}')
    # fig.savefig(f'results/{desc}_large.png')
    if show:
        fig.show()
    return fig, ax

def SpectralDP(G, pos, tags, N_communities, desc='Karate', **kwargs):

    laplacian = scipy.sparse.csr_matrix.todense(nx.normalized_laplacian_matrix(G, weight=1))
    u, s, v = scipy.linalg.svd(laplacian)
    # fig, ax = plt.subplots(figsize=(15, 6))
    # sns.scatterplot(x=np.arange(len(s)), y=s[::-1], ax=ax)
    # ax.set_title('Graph Spectrum')
    # fig.show()

    u = u[:, -N_communities:-1]  # truncated
    delta_dist = pairwise_distance(len(tags) - 1, lambda x, y: np.linalg.norm(u[x] - u[y]))

    # hop_dist = pairwise_distance(len(tags) - 1, lambda x, y: shortest_path_len(G, x, y))

    in_data = np.array([pos[i] for i in range(len(tags))])
    DP = DPClustering(in_data, tags)
    DP.run(rho_dist=delta_dist, delta_dist=delta_dist, max_clusters=N_communities, **kwargs)
    # run_fig.savefig(f'results/{desc}.png')
    #
    entropy, purity = DP.eval()
    fig, _ = cluster_plot(G, pos, tags, [DP.align[_] if _ != -1 else -1 for _ in DP.pred_tags], desc=f'DP: entropy={entropy:.3f}, purity={purity:.3f}', **kwargs)
    fig.savefig(f'results/{desc}_large.png')

    return DP, u

def paper_cluster():
    """
    data: can use gamma=True for hint
    'spiral', 8, 5
    'jain', 11, 10
    'flame', 4, 6
    'aggregation', 12, 6
    """
    data, rho_threshold, delta_threshold = 'spiral', 10, 5
    # data, rho_threshold, delta_threshold = 'jain', 13, 10
    # data, rho_threshold, delta_threshold = 'flame', 9, 6
    # data, rho_threshold, delta_threshold = 'aggregation', 16, 5

    pos = np.loadtxt(f'data/{data}.txt')
    tags = [int(_)-1 for _ in pos[:, 2]]
    pos = pos[:, :2]
    # dist_dict = {(int(i)-1, int(j)-1): d for i, j, d in np.loadtxt('data/spiral_distance.txt')}
    dist_dict = pairwise_distance(len(pos) - 1, lambda x, y: np.linalg.norm(pos[x] - pos[y]))

    N_communities = len(set(tags))

    dp = DPClustering(pos, tags)
    dp.run(rho_dist=dist_dict,
           delta_dist=dist_dict,
           max_clusters=N_communities,
           rho_threshold=rho_threshold,
           delta_threshold=delta_threshold,
           gamma=True,
           gauss=True)
    entropy, purity = dp.eval()
    print(f'DP: entropy={entropy:.3f}, purity={purity:.3f}')

    dp.plot(show=True, threshold=True, desc=f'DP: entropy={entropy:.3f}, purity={purity:.3f}')

    # compare to k-means
    dp.k_means(pos, N_communities)
    entropy, purity = dp.eval()
    dp.plot(show=True, threshold=False, desc=f'K-means: entropy={entropy:.3f}, purity={purity:.3f}')

def rand_cluster():
    G = nx.random_partition_graph([5, 10, 15, 15, 15], 0.7, 0.1, seed=0)
    # G = nx.planted_partition_graph(5, 10, 0.5, 0.1, seed=0)
    tags = [G.nodes[v]['block'] for v in G]

    # for plots only
    for e in G.edges:
        G.edges[e]['weight'] = 1 if tags[e[0]] == tags[e[1]] else 0.4

    N_communities = len(set(tags))
    pos = nx.spring_layout(G)

    # Spectral Density Peak Clustering
    dp, embed = SpectralDP(G, pos, tags, N_communities, desc='rand',
                           gamma=False, rho_threshold=1, delta_threshold=0.2)
    print(dp.eval())

    entropy, purity = dp.eval()
    fig, ax = dp.plot(show=False, threshold=False, desc=f'DP: entropy={entropy:.3f}, purity={purity:.3f}')
    add_edges(ax, G, pos)
    fig.show()
    fig.savefig(f'results/rand_dp.png')

    # compare to k-means
    dp.k_means(embed, N_communities)
    entropy, purity = dp.eval()
    fig, ax = dp.plot(show=False, threshold=False, desc=f'K-means: entropy={entropy:.3f}, purity={purity:.3f}')
    add_edges(ax, G, pos)
    fig.show()
    fig.savefig(f'results/rand_k_means.png')

    # compare to k-means
    dp.k_means(embed, N_communities)
    entropy, purity = dp.eval()
    fig, _ = cluster_plot(G, pos, tags, [dp.align[_] for _ in dp.pred_tags], desc=f'K-means: entropy={entropy:.3f}, purity={purity:.3f}')
    fig.savefig(f'results/rand_k_means_large.png')

def karate_cluster():
    """
    Karate Club network (2 clusters)
    """
    G = nx.karate_club_graph()
    tags = [0 if G.nodes[v]['club'] == 'Mr. Hi' else 1 for v in G]

    N_communities = len(set(tags))
    pos = nx.spring_layout(G)

    # Spectral Density Peak Clustering
    dp, embed = SpectralDP(G, pos, tags, N_communities, desc='karate',
                           gamma=True, rho_threshold=5, delta_threshold=0.15)
    print(dp.eval())

    entropy, purity = dp.eval()
    fig, ax = dp.plot(show=False, threshold=False, desc=f'DP: entropy={entropy:.3f}, purity={purity:.3f}')
    add_edges(ax, G, pos)
    fig.show()
    fig.savefig(f'results/karate_dp.png')

    # compare to k-means
    dp.k_means(embed, N_communities)
    entropy, purity = dp.eval()
    fig, ax = dp.plot(show=False, threshold=False, desc=f'K-means: entropy={entropy:.3f}, purity={purity:.3f}')
    add_edges(ax, G, pos)
    fig.show()
    fig.savefig(f'results/karate_k_means.png')

    # compare to k-means
    dp.k_means(embed, N_communities)
    entropy, purity = dp.eval()
    fig, _ = cluster_plot(G, pos, tags, [dp.align[_] for _ in dp.pred_tags], desc=f'K-means: entropy={entropy:.3f}, purity={purity:.3f}')
    fig.savefig(f'results/karate_k_means_large.png')

def email_cluster():
    with open('data/email_edges.txt', 'rb') as f:
        G = nx.read_adjlist(f, create_using=nx.DiGraph())
    G = G.to_undirected(reciprocal=True)
    G.remove_edges_from(nx.selfloop_edges(G))
    G.remove_nodes_from(list(nx.isolates(G)))

    str_nodes = list(G.nodes)
    str_nodes.sort(key=lambda x: int(x))
    str_nodes_int = {k: v for v, k in enumerate(str_nodes)}
    G = nx.relabel_nodes(G, str_nodes_int)

    communities = {}
    with open('data/email_labels.txt') as f:
        c, tag_set = 0, {}
        for l in f.readlines():
            k, v = l.split(' ')
            if k in str_nodes_int:
                if v not in tag_set:
                    tag_set[v] = c
                    c += 1
                communities[str_nodes_int[k]] = tag_set[v]  # int(v[:-1])
    N_communities = len(set(communities.values()))

    # for plots only
    for e in G.edges:
        G.edges[e]['weight'] = 0.05 if communities[e[0]] == communities[e[1]] else 0.001

    pos = nx.spring_layout(G)
    tags = [communities[v] for v in G]

    # Spectral Density Peak Clustering
    dp, embed = SpectralDP(G, pos, tags, N_communities, desc='email', node_size=10, label=False, plot_edge=False,
                           gamma=True,  percent=1., gauss=True, cut_gauss=True)
    print(dp.eval())

    entropy, purity = dp.eval()
    fig, ax = dp.plot(show=False, threshold=False, desc=f'DP: entropy={entropy:.3f}, purity={purity:.3f}')
    fig.show()
    fig.savefig(f'results/email_dp.png')

    # compare to k-means
    dp.k_means(embed, N_communities)
    entropy, purity = dp.eval()
    fig, ax = dp.plot(show=False, threshold=False, desc=f'K-means: entropy={entropy:.3f}, purity={purity:.3f}')
    fig.show()
    fig.savefig(f'results/email_k_means.png')

    # compare to k-means
    dp.k_means(embed, N_communities)
    entropy, purity = dp.eval()
    fig, _ = cluster_plot(G, pos, tags, [dp.align[_] for _ in dp.pred_tags],
                          desc=f'K-means: entropy={entropy:.3f}, purity={purity:.3f}', node_size=10, label=False)
    fig.savefig(f'results/email_k_means_large.png')

def facebook_cluster():
    """
    too large...
    :return:
    """
    with open('data/facebook_combined.txt', 'rb') as f:
        G = nx.read_adjlist(f)

    # relabel
    str_nodes = list(G.nodes)
    str_nodes.sort(key=lambda x: int(x))
    str_nodes_int = {k: v for v, k in enumerate(str_nodes)}
    G = nx.relabel_nodes(G, str_nodes_int)

    with open('data/fb_tags.txt', 'r') as f:
        content = f.read()
        communities = eval(content)

    N_communities = len(set(communities.values()))

    # for plots only
    for e in G.edges:
        G.edges[e]['weight'] = 0.05 if communities[e[0]] == communities[e[1]] else 0.001

    pos = nx.spring_layout(G)
    tags = [communities[v] for v in G]

    # Spectral Density Peak Clustering
    dp, embed = SpectralDP(G, pos, tags, N_communities, desc='facebook', node_size=10, label=False, plot_edge=False,
                           gamma=True, percent=2., gauss=True, cut_gauss=True)
    print(dp.eval())

    entropy, purity = dp.eval()
    fig, ax = dp.plot(show=False, threshold=False, desc=f'DP: entropy={entropy:.3f}, purity={purity:.3f}')
    fig.show()
    fig.savefig(f'results/facebook_dp.png')

    # compare to k-means
    dp.k_means(embed, N_communities)
    entropy, purity = dp.eval()
    fig, ax = dp.plot(show=False, threshold=False, desc=f'K-means: entropy={entropy:.3f}, purity={purity:.3f}')
    fig.show()
    fig.savefig(f'results/facebook_k_means.png')

    # compare to k-means
    dp.k_means(embed, N_communities)
    entropy, purity = dp.eval()
    fig, _ = cluster_plot(G, pos, tags, [dp.align[_] for _ in dp.pred_tags],
                          desc=f'K-means: entropy={entropy:.3f}, purity={purity:.3f}', node_size=10, label=False)
    fig.savefig(f'results/facebook_k_means_large.png')

if __name__ == '__main__':
    karate_cluster()


from socket import SO_REUSEPORT, SOL_SOCKET, socket, AF_INET, SOCK_DGRAM, SO_BROADCAST
from helpers import *
from packet import *
from threading import Thread
from router import udprouter
import netifaces as ni
from sys import argv

# Class inherits methods from udprouter

class udphost(udprouter):

    # Super init is not called as not all udprouter threads are needed
    def __init__(self, id: int, ip: str):
        """
        Constructor for udphost class
        """
        self.id = id
        self.ip = ip
        self.rt = routing_table()
        self.seq = 0

        # Start thread for broadcasting HELLO packets
        Thread(
            target=self.broadcast_hello_packet, args=(self.id, DEFAULT_TTL)).start()

        # Start thread for handling and forwarding HELLO packets
        Thread(target=self.handle_hello_packet).start()

        sleep(5)
        # Start thread for handling multicast data sending and receiving
        Thread(target=self.handle_centroid_packets).start()

    # Method to handle sending and receiving of multicast data
    def handle_centroid_packets(self):
        """
        Method to handle receiving and forwarding of centroid request/reply packets
        """

        # Loop to allow user to keep sending and receiving data
        while True:

            # Ask if user wants to send or receive from this host
            print("Would you like to send or receive data? (s/r)")
            choice = input()

            # If user wants to send
            if (choice == 's'):

                # Ask user for destination host(s)
                hosts = input("To what hosts would you like to send to?")

                # Split input into list of hosts
                hosts = hosts.split(",")
                inthosts = []
                for i in range(0, len(hosts)):
                    inthosts.append(int(hosts[i]))

                # If host is not in routing table, go back to start of loop
                for dst in inthosts:
                    if not self.rt.check_entry(dst):
                        print("Host not in routing table")
                        continue

                # If only one host, send to that host as unicast
                if len(inthosts) == 1:

                    # Set up socket
                    data_sock = socket(AF_INET, SOCK_DGRAM)
                    data_sock.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)

                    # Loop to send data
                    while True:

                        # Ask user for data to send
                        input_msg = input(
                            "Enter message to send to destinations: \n")

                        # IDK if this is used
                        print("Enter 'stop' to stop sending reset")

                        # Create packet
                        data_packet = create_data_packet(
                            pkttype=1, seq=self.seq, src=self.id, dst=inthosts[0], data=input_msg)

                        # Print where the packet is being sent to
                        print("Sending data packet to ",
                              self.rt.get_next_hop(inthosts[0]))

                        # Send packet
                        data_sock.sendto(
                            data_packet, (self.rt.get_next_hop(inthosts[0]), DATA_PORT))

                        # If the user wants to stop sending, break out of loop
                        if input_msg == "end_of_transmission":
                            data_packet = create_data_packet(
                                pkttype=2, seq=self.seq, src=self.id, dst=inthosts[0], data="end_of_transmission")
                            data_sock.sendto(
                                data_packet, (self.rt.get_next_hop(inthosts[0]), DATA_PORT))
                            break

                    # Close socket
                    data_sock.close()

                elif len(inthosts) >= 2:  # If multiple hosts, send to all as multicast

                    # Create packet to fit the number of destinations
                    if (len(inthosts) == 2):
                        centroid_request_packet = create_centroid_request_packet(
                            self.seq, self.id, inthosts[0], inthosts[1])
                    else:
                        centroid_request_packet = create_centroid_request_packet(
                            self.seq, self.id, inthosts[0], inthosts[1], inthosts[2])

                    # Set up socket, send, and close socket for each broadcast address
                    for addr in self.broadcast_addresses:
                        sock = socket(AF_INET, SOCK_DGRAM)
                        sock.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
                        sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
                        sock.bind((addr, CENTROID_SETUP_PORT))
                        print(addr)
                        sock.sendto(centroid_request_packet,
                                    (addr, CENTROID_SETUP_PORT))
                        sock.close()

                    # wait for centroid reply
                    centsock = socket(AF_INET, SOCK_DGRAM)
                    centsock.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
                    centsock.bind(('0.0.0.0', CENTROID_SETUP_PORT))
                    packet, addr = centsock.recvfrom(10)

                    # decode data packet
                    centroid, dst = decode_centroid_reply_packet(packet)

                    # send data packet to centroid
                    data_sock = socket(AF_INET, SOCK_DGRAM)
                    data_sock.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
                    while True:
                        input_msg = input(
                            "Enter message to send to destinations: ")
                        data_packet = create_data_packet(
                            pkttype=1, seq=self.seq, src=self.id, dst=centroid, data=input_msg)

                        print("Sending data packet to ",
                              self.rt.get_next_hop(centroid))
                        # data_sock.bind((self.rt.get_next_hop(centroid), DATA_PORT))
                        data_sock.sendto(
                            data_packet, (self.rt.get_next_hop(centroid), DATA_PORT))
                        print('Sent data packet to centroid', data_packet)
                        if input_msg == 'end_of_transmission':
                            break
                    data_sock.close()

            elif choice == 'r':  # If user wants to receive

                # Set up socket
                sock = socket(AF_INET, SOCK_DGRAM)
                sock.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
                sock.bind(('0.0.0.0', DATA_PORT))

                # Loop to receive data
                while True:
                    data_packet, addr = sock.recvfrom(1024)
                    pkttype, seq, src, dst, data = decode_data_packet(
                        data_packet)
                    print('Received data packet from ' +
                          str(src) + ' to ' + str(dst))
                    if data != 'end_of_transmission':
                        print('Data packet data: ' + data)
                    else:  # If data is end of transmission, break out of loop
                        print('End of transmission')
                        break
            else:
                print("Invalid input")

if __name__ == '__main__':
    print("Router Started...")

    # Print all interfaces
    print(ni.interfaces())

    # Get the ip address to be used on print displays to the user
    device_ip: str = ''
    for ifstring in ni.interfaces():
        if 'eth' in ifstring:
            device_ip = ni.ifaddresses(ifstring)[ni.AF_INET][0]['addr']
            break

    # Take device ID as command line argument, must be between 1 and 254
    device_id = 0
    if (len(argv) > 1 and argv[1].isdigit() and int(argv[1]) >= 1 and int(argv[1]) <= 254):
        device_id = int(argv[1])
    else:
        print("Invalid device ID. Must be an integer between 1 and 254")
        exit(1)

    # Print the device IP and ID to the user
    print("Device IP: ", device_ip)
    print("Device ID: ", device_id)

    # Create and run router object
    udp_host = udphost(id=(device_id), ip=device_ip)


from collections import deque

def main():

    def musicProgram():
        result = []
        queue = deque()

        for i in range(1, N + 1):
            if indegree[i] == 0:
                queue.append(i)

        while queue:
            currS = queue.popleft()
            result.append(currS)
            for i in singer[currS]:
                indegree[i] -= 1
                if indegree[i] == 0:
                    queue.append(i)
        return result

    N, M = map(int, input().split())

    singer = [[] for _ in range(N + 1)]
    indegree = [0] * (N + 1)
    for _ in range(M):
        PD = list(map(int, input().split()))
        for s in range(1, PD[0]):
            before, after = PD[s], PD[s + 1]
            singer[before].append(after)
            indegree[after] += 1

    sequence = musicProgram()
    print(0) if len(sequence) != N else print(*sequence, sep='\n')

if __name__ == "__main__":
    main()


# Messing with Pie Charts
# Author: Megan O'Donovan

import numpy as np
import matplotlib.pyplot as plt

fruit = np.array(['Apple', 'Orange', 'Banana'])
numbers = np.array([23,77,50])

plt.pie(numbers, labels = fruit)
plt.legend()
plt.show()


# -*- coding: utf-8 -*-

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""
Produce skeleton Performance Data Review Requests.

This was mostly copies from glean_parser, and should be kept in sync.
"""

import re
from pathlib import Path
from typing import Sequence

from glean_parser import parser, util

def generate(
    bug: str,
    metrics_files: Sequence[Path],
) -> int:
    """
    Commandline helper for Data Review Request template generation.

    :param bug: pattern to match in metrics' bug_numbers lists.
    :param metrics_files: List of Path objects to load metrics from.
    :return: Non-zero if there were any errors.
    """

    metrics_files = util.ensure_list(metrics_files)

    # Accept any value of expires.
    parser_options = {
        "allow_reserved": True,
        "custom_is_expired": lambda expires: False,
        "custom_validate_expires": lambda expires: True,
    }
    all_objects = parser.parse_objects(metrics_files, parser_options)

    if util.report_validation_errors(all_objects):
        return 1

    # I tried [\W\Z] but it complained. So `|` it is.
    reobj = re.compile(f"\\W{bug}\\W|\\W{bug}$")
    durations = set()
    responsible_emails = set()
    metrics_table = ""
    for category_name, metrics in all_objects.value.items():
        for metric in metrics.values():
            if not any([len(reobj.findall(bug)) == 1 for bug in metric.bugs]):
                continue

            metric_name = util.snake_case(metric.name)
            category_name = util.snake_case(category_name)
            one_line_desc = metric.description.replace("\n", " ")
            sensitivity = ", ".join([s.name for s in metric.data_sensitivity])
            last_bug = metric.bugs[-1]
            metrics_table += f"`{category_name}.{metric_name}` | "
            metrics_table += f"{one_line_desc} | {sensitivity} | {last_bug}\n"
            if metric.type == "event" and len(metric.allowed_extra_keys):
                for extra_name, extra_detail in metric.extra_keys.items():
                    extra_one_line_desc = extra_detail["description"].replace("\n", " ")
                    metrics_table += f"`{category_name}.{metric_name}#{extra_name}` | "
                    metrics_table += (
                        f"{extra_one_line_desc} | {sensitivity} | {last_bug}\n"
                    )

            durations.add(metric.expires)

            if metric.expires == "never":
                responsible_emails.update(metric.notification_emails)

    if len(durations) == 1:
        duration = next(iter(durations))
        if duration == "never":
            collection_duration = "This collection will be collected permanently."
        else:
            collection_duration = f"This collection has expiry '{duration}'"
    else:
        collection_duration = "Parts of this collection expire at different times: "
        collection_duration += f"{durations}"

    if "never" in durations:
        collection_duration += "\n" + ", ".join(responsible_emails) + " "
        collection_duration += "will be responsible for the permanent collections."

    if len(durations) == 0:
        print(f"I'm sorry, I couldn't find metrics matching the bug number {bug}.")
        return 1

    # This template is pulled from
    # https://github.com/mozilla/data-review/blob/main/request.md
    print(
        """
!! Reminder: it is your responsibility to complete and check the correctness of
!! this automatically-generated request skeleton before requesting Data
!! Collection Review. See https://wiki.mozilla.org/Data_Collection for details.

DATA REVIEW REQUEST
1. What questions will you answer with this data?

TODO: Fill this in.

2. Why does Mozilla need to answer these questions? Are there benefits for users?
   Do we need this information to address product or business requirements?

In order to guarantee the performance of our products, it is vital to monitor
real-world installs used by real-world users.

3. What alternative methods did you consider to answer these questions?
   Why were they not sufficient?

Our ability to measure the practical performance impact of changes through CI
and manual testing is limited. Monitoring the performance of our products in
the wild among real users is the only way to be sure we have an accurate
picture.

4. Can current instrumentation answer these questions?

No.

5. List all proposed measurements and indicate the category of data collection for each
   measurement, using the Firefox data collection categories found on the Mozilla wiki.

Measurement Name | Measurement Description | Data Collection Category | Tracking Bug
---------------- | ----------------------- | ------------------------ | ------------"""
    )
    print(metrics_table)
    print(
        """
6. Please provide a link to the documentation for this data collection which
   describes the ultimate data set in a public, complete, and accurate way.

This collection is Glean so is documented
[in the Glean Dictionary](https://dictionary.telemetry.mozilla.org).

7. How long will this data be collected?
"""
    )
    print(collection_duration)
    print(
        """
8. What populations will you measure?

All channels, countries, and locales. No filters.

9. If this data collection is default on, what is the opt-out mechanism for users?

These collections are Glean. The opt-out can be found in the product's preferences.

10. Please provide a general description of how you will analyze this data.

This will be continuously monitored for regression and improvement detection.

11. Where do you intend to share the results of your analysis?

Internal monitoring (GLAM, Redash, Looker, etc.).

12. Is there a third-party tool (i.e. not Telemetry) that you
    are proposing to use for this data collection?

No.
"""
    )

    return 0


# 迭代器
## 定义一个迭代器
from collections.abc import Iterable

# 如果要对list实现类似Java那样的下标循环怎么办？Python内置的enumerate函数可以把一个list变成索引-元素对，这样就可以在for循环中同时迭代索引和元素本身
lista = [1, 2, 5, 7, 9, 2]
for index, value in enumerate(lista):
    print(f"索引值是：{index}", f"当前索引对应的值是：{value}")
# it = iter(lista)
# print(type(it))
# print(next(it))
# print(next(it))
'''for 循环遍历迭代器'''
# for i in it:
#     print(i, end=" ")
"""while 循环遍历迭代器"""
# n = len(lista)
# while n > 0:
#     print(next(it), end=" ")
#     n -= 1
a = {"name": "zhangsan", "data": 1, "msg": "ok"}
if isinstance(a, Iterable):
    print("这是一个可迭代对象")
    for x, y in a.items():
        print(x, y)
else:
    print("这不是一个可迭代对象")

import os

newlist = [d for d in os.listdir('../../')]
print(newlist)


import pandas as pd
from pandas._libs import missing
import requests
from datetime import datetime, timedelta
import time

authors_papers_csv = "C:/Users/zeagle/scratch/Research/authors_papers.csv"
external_id_csv = "C:/Users/zeagle/scratch/Research/external_ids.csv"
papers = pd.read_csv(authors_papers_csv)

requestTiming = pd.DataFrame(columns=['request','timestamp'])
rateLimit = 75
#rate limit time in minutes
rateTime = 5

def checkRate():
    now = datetime.now()
    rateLimitBeginning = pd.Timestamp(now - timedelta(hours=0, minutes=rateTime,seconds=0))
    filtermask = requestTiming['timestamp'] > rateLimitBeginning
    filteredDf = requestTiming[filtermask]
    return filteredDf.shape[0]

def timeRemaining():
    now = datetime.now()
    rateLimitBeginning = pd.Timestamp(now - timedelta(hours=0, minutes=rateTime,seconds=0))
    filtermask = requestTiming['timestamp'] > rateLimitBeginning
    filteredDf = requestTiming[filtermask]
    sortedDf = filteredDf.sort_values(by='timestamp', ascending=True, na_position='last')
    timeleft = (now - sortedDf.iloc[1]['timestamp']).seconds
    timeleft = rateTime*60 - timeleft
    return timeleft

def open_exisiting_file(file):
    try:
        exisiting_list = pd.read_csv(file, dtype="str")
    except FileNotFoundError:
        exisiting_list = pd.DataFrame()
    return exisiting_list

def save_file(new_item, existing_list, file):
    paper_list_to_save = existing_list.append(new_item, sort=False, ignore_index=True)
    paper_list_to_save = paper_list_to_save.drop_duplicates(ignore_index=True)
    paper_list_to_save.to_csv(file, index=False)

existing_ids = open_exisiting_file(external_id_csv)

#get list of papers from semantic scholar list
paper_list = open_exisiting_file(authors_papers_csv)
paper_list = paper_list[['paperId',]]
paper_list = paper_list.dropna()
print(paper_list)
#get existing list of cross ref
id_list = open_exisiting_file(external_id_csv)
id_list = id_list[['paperId',]]
print(id_list)

combined_list = pd.merge(paper_list, id_list, how='outer', indicator="Exists", left_on='paperId', right_on='paperId')
print(combined_list[combined_list['Exists'] != 'both'])
missing_details = combined_list[combined_list['Exists'] != 'both']
print(missing_details)
combined_list.to_csv('C:/Users/zeagle/scratch/Research/test.csv')
paperIds = missing_details['paperId']
print(paperIds)

externalIds_list = pd.DataFrame()
for paper in paperIds:
    request = 'https://api.semanticscholar.org/graph/v1/paper/'+paper+'?fields=externalIds'
    now = datetime.now()
    requestTiming = requestTiming.append({
        "request": request,
        "timestamp": pd.Timestamp(now.strftime("%m/%d/%Y %H:%M:%S"))
        }, ignore_index=True)
    while checkRate() > rateLimit:
        print("Sleeping for rate limit. Time remaining:" + str(timeRemaining()), end='\r')
        time.sleep(1)
    paperInfo = requests.get(request).json()
    externalIds = paperInfo['externalIds']
    externalIds["paperId"] = paper
    #externalIds_list = externalIds_list.append(externalIds, ignore_index=True)
    print(externalIds)
    existing_ids = open_exisiting_file(external_id_csv)
    save_file(externalIds, existing_ids, external_id_csv)


#! /usr/bin/env python3

from scipy.stats import pearsonr, norm
import statsmodels.regression.linear_model as lm
import numpy as np #type: ignore
import pandas as pd #type: ignore
from typing import List, Dict, Tuple, Optional
from collections import namedtuple

def weighted_cov(x: np.array, y: np.array, w: np.array) -> float:
    """Weighted covariance between vectors x and y, with weights w
    Args:
        x (np.array): numerical vector x
        y (np.array): numerical vector y
        w (np.array): weights for calculating weighted average of x
    Returns:
        (float): weighted covariance
    """
    return np.average( ( (x-np.average(x, weights=w) ) * (y - np.average(y, weights=w) ) ) , weights=w )

def weighted_pearsonr(x: np.array, y: np.array, w: np.array) -> float:
    return weighted_cov(x, y, w) / np.sqrt( weighted_cov(x, x, w) * weighted_cov(y, y, w) )

def calculate_r2(dataset: pd.DataFrame, x_label: str, y_label: str, stderr_label: str) -> Tuple[float, float, int, int]:
    """Calculate r2 values for dataset
    Args:
        dataset (pd.DataFrame): dataset
        x_label (str): x values column label in dataset
        y_label (str): y values column label in dataset
        stderr_label (str): standard error column label in dataset
    Returns:
        (Tuple[float,float,int,int]): Tuple of values: r^2 without weights, r^2 weighted by inverse stderr, N of r^2 calculation, N of r^2 weighted calculation
    """
    data_w=dataset[[x_label,y_label,stderr_label]].copy()
    data_r2 = data_w[[x_label,y_label]].copy()
    data_r2=data_r2.dropna(how="any")
    data_w=data_w.dropna(how="any")
    r_2=np.nan
    r_w=np.nan
    N_r=np.nan
    N_w=np.nan
    if data_r2.shape[0]>= 2:
        x_array = data_r2[x_label].values
        y_array = data_r2[y_label].values
        r,_=pearsonr(x_array,y_array)
        r_2=r**2
        N_r=data_r2.shape[0]
    if data_w.shape[0]>=2:
        x_array = data_w[x_label].values
        y_array = data_w[y_label].values
        stderr = data_w[stderr_label].values
        weight_array= 1/(stderr**2 + 1e-9) #weights as inverse of variance
        r_w=weighted_pearsonr(x_array,y_array,weight_array)
        r_w=r_w**2
        N_w=data_w.shape[0]
    return (r_2,r_w,N_r,N_w)

RegressionResults = namedtuple('RegressionResults',['slope', 'stderr', 'tstat', 'pval', 'rsquared'])

def calculate_regression(x: np.array,
                        y: np.array,
                        weights: Optional[np.array] = None)-> RegressionResults :
    """Calculate regression y=bx coefficients from data.
    Model uses statsmodels.regression.linear_model.WLS.
    Args:
        x (np.array): numpy array of x-coordinates
        y (np.array): numpy array of y-coordinates
        weights (Optional[np.array]): numpy array of point weights. These are passed directly to WSL.
    Returns:
        (RegressionResults): Named tuple with variables slope, stderr, t-statistic, pvalue, rsquared
    """
    weights = weights if type(weights) != type(None) else 1.0
    model = lm.WLS(y,x,weights)
    results = model.fit()
    slope=results.params[0]
    stderr = results.bse[0]
    tstat = results.tvalues[0]
    pval = results.pvalues[0]
    rsquared = results.rsquared_adj

    return RegressionResults(slope, stderr, tstat, pval,rsquared)


"""Menu models"""

# Django
from django.db import models

class Option(models.Model):
    """Option model."""

    meal = models.CharField(
        'meal name',
        max_length=255,
        help_text=('meal option')
    )

class Menu(models.Model):
    """Menu model.

    Specific meals create a complete menu.
    """

    title_text = models.TextField(
        blank=True,
        null=True,
        help_text=(
            'greeting or menu description'
        )
    )

    option_meals = models.ManyToManyField(
        to='option',
        related_name='meals',
    )

    agreed_date = models.DateField(null=True, blank=True, default=None)

    create_date = models.DateTimeField(auto_now_add=True)

    updated_date = models.DateTimeField(auto_now=True)

    def __str__(self) -> str:
        """return username."""
        return self.agreed_date

    def get_short_name(self) -> str:
        """Return usermane."""
        return self.agreed_date


import torch
from torch import nn

class NonLocalAttention(nn.Module):
    """
        Attention from all positions in object B to all positions in object A
    """
    def __init__(self, in_channels=256, inter_channels=None, bn_layer=True):
        super(NonLocalAttention, self).__init__()

        self.in_channels = in_channels
        self.inter_channels = inter_channels

        # Set default inter_channels
        if self.inter_channels is None:
            self.inter_channels = in_channels // 2
            if self.inter_channels == 0:
                self.inter_channels = 1

        self.g = nn.Conv2d(in_channels=self.in_channels, out_channels=self.inter_channels,
                         kernel_size=1, stride=1, padding=0)

        if bn_layer:
            self.W = nn.Sequential(
                nn.Conv2d(in_channels=self.inter_channels, out_channels=self.in_channels,
                        bias=False, kernel_size=1, stride=1, padding=0),
                nn.BatchNorm2d(self.in_channels)
            )
        else:
            self.W = nn.Conv2d(in_channels=self.inter_channels, out_channels=self.in_channels,
                    kernel_size=1, stride=1, padding=0)

        self.theta = nn.Conv2d(in_channels=self.in_channels, out_channels=self.inter_channels,
                             kernel_size=1, stride=1, padding=0)

        self.phi = nn.Conv2d(in_channels=self.in_channels, out_channels=self.inter_channels,
                           kernel_size=1, stride=1, padding=0)

        # Initialize weights
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight.data, mode="fan_in")
                if m.bias is not None:
                    m.bias.data.zero_()

    def forward(self, a, b):
        B, C, H, W = a.size()

        # Pairwise relationship
        theta_a = self.theta(a).reshape(B, self.inter_channels, -1).permute(0, 2, 1)
        phi_b = self.phi(b).reshape(B, self.inter_channels, -1)
        # Correlation of size (B, H * W, H * W)
        f = torch.matmul(theta_a, phi_b)
        f_div_C = f / f.size(-1)

        # Get representation of b
        g_b = self.g(b).view(B, self.inter_channels, -1)
        g_b = g_b.permute(0, 2, 1)

        # Combine
        y = torch.matmul(f_div_C, g_b)
        y = y.permute(0, 2, 1).contiguous()
        y = y.view(B, self.inter_channels, *a.size()[2:])
        W_y = self.W(y)

        return W_y


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import os
import json

class PoetryCrawlPipeline(object):
    def open_spider(self, spider):
        print("crawl start......")

    def close_spider(self, spider):
        print("crawl complete!")

    def process_item(self, item, spider):
        # convert to dict first
        i = dict(item)

        # dist/ is the output dir
        if not os.path.exists('./dist'):
            os.mkdir('./dist')

        dir_dynasty = './dist/' + i['dynasty']
        dir_file = './dist/' + i['dynasty'] + '/' + i['dynasty'] + '_' + i['author'] + '.json'

        if not os.path.exists(dir_dynasty):
            os.makedirs(dir_dynasty)

        if not os.path.exists(dir_file):
            f = open(dir_file, 'w+', encoding='utf-8')
            text = {
                "dynasty": i["dynasty"],
                "author": i["author"],
                "intro": i["intro"],
                "items": [
                    {
                        "title": i["title"],
                        "content": i["content"]
                    }
                ]
            }
            json.dump(text, f, ensure_ascii=False, indent=4)
            f.close()

        else:
            f = open(dir_file, 'r+', encoding='utf-8')
            text = json.load(f)
            text["items"].append(
                {
                    "title": i["title"],
                    "content": i["content"]
                }
            )
            f.close()

            f = open(dir_file, 'w+', encoding='utf-8')
            json.dump(text, f, ensure_ascii=False, indent=4)
            f.close()

        # stdout info
        print("已处理：" + i['dynasty'] + "_" + i['author'] + "_" + i["title"])

        return item


# -*- coding: UTF-8 -*-

from db_entities.message.message import Message
from db_entities.message.rcp_list import Rcp_list
from sendmail.post_office import PostOffice
from google.appengine.ext import db
from tools.tag_processor import tag_processor

class Handler():

    def send(self, owner, rcp, title, body): #rcp must be a list!
        new_msg = Message()
        new_msg.owner = owner
        new_msg.title = tag_processor().mask_tags(title)
        new_msg.body = tag_processor().prepare(body)
        message = new_msg.put()

        for r in rcp:
            msg_r = Rcp_list()
            msg_r.rcp = r
            msg_r.message = message
            msg_r.put()

#        to = [r for r in rcp]  # generate list of rcp keys

        PostOffice().append_to_queue(rcp, title, body)

    def get_incoming(self, user):
        rcp_list = Rcp_list.gql("where rcp = :rcp", rcp = user)
        messages = []
        for entity in rcp_list:
            if entity.deleted: continue
            messages.append(entity.message)
        return self.__get_letters(messages)

    def get_outgoing(self, user):
        messages = Message.gql("where owner = :owner and deleted = :is_deleted", owner = user, is_deleted = False)
        return self.__get_letters(messages)

    def __get_letters(self, messages):
        formalized_messages = []
        for message in messages:
            owner = { 'login' : message.owner.login }
            rcp = []
            r_read = False
            rcp_list = Rcp_list.gql("where message = :message", message = message)
            for r in rcp_list:
                rcp.append({'login': r.rcp.login})
                r_read = r.read
            title = message.title
            o_read = message.read
            datetime = message.datetime
            formalized_messages.append({'key' : message.key(), 'r_read' : r_read, 'o_read' : o_read, 'owner':owner, 'rcp':rcp[:5], 'title':title[:32] + " ...", 'datetime': datetime})
            formalized_messages.sort(key=lambda k: k['datetime'])
        return formalized_messages

    def get_letter(self, letter, user):
        letter_ = db.get(letter)
        rcp = False
        try:
            rcp = Rcp_list.gql("where message = :letter and rcp = :user", letter = db.Key(letter), user = user)[0]
        except: pass
        if letter_.owner.key() == db.Key(user) or rcp:
            if rcp:
                rcp.read = True
                rcp.put()
            return letter_

    def delete(self, messages, user):
        messages_ = db.get(messages)
        for message in messages_:
            if message.owner == user:
                message.deleted = True
                message.read = True
                message.put()
            else:
                rcp = Rcp_list.gql("where message = :message and rcp = :rcp", message = message, rcp = user)[0]
                rcp.deleted = True
                rcp.read = True
                rcp.put()

    def has_unread_msg(self, user):
        try:
            return Rcp_list.gql("where rcp = :rcp and read = :read", rcp = user, read = False).count(1)
        except:
            return False


from plotly.subplots import make_subplots
import plotly.graph_objects as go
import pandas as pd
import numpy as np
import scipy
import plotly.express as px
from app_main.constants import color_types, color_types_title, font_size, marker_size
from operations.preprocessing import preprocess
from operator import itemgetter
from app_main.utilities import short_ephys_labels, short_morph_labels
import math

twopi = 2 * math.pi

def plot_id(r, c):
    """ Compute a unique 1-based index for each plot based on its 1-based row and column"""
    return 2 * (r - 1) + c

def normalize_raw_df(data):
    total = np.sum(data, axis=1)  # sum measures across columns
    #total = np.array(total)  # convert from matrix to vector
    X = np.log10(data / total[:, None] * np.median(total) + 1)
    X = np.array(X)
    X = X - X.mean(axis=0)
    X = X / np.std(X, axis=0)
    return X

def create_bibiplot1x2(data_1, data_2, d1, d2, x_col, y_col, dataset, color, metadata_type,
                       preprocess_1, preprocess_2, label_1, label_2, num_features, size_key='default'):
    """
    Create 1x2 bibiplot
    :param data_1: raw dataset 1 (dataframe)
    :param data_2: raw dataset 2 (dataframe)
    :param d1: projection of dataset 1 into latent space
    :param d2: projection of dataset 2 into latent space
    :param x_col: latent space component used as the x plotting dimension
    :param y_col: latent space component used as the y plotting dimension
    :param dataset: which dataset selected by user (visual/motor/upload)
    :param color: vector identifying groups for coloring plots
    :param metadata_type: name of metadata column to use
    :param preprocess_1: user-selected preprocessing method for dataset 1
    :param preprocess_2: user-selected preprocessing method for dataset 2
    :param label_1: label for dataset 1
    :param label_2: label for dataset 2
    :param num_features: number of top correlated features to display
    :param size_key: key to use in selecting font sizes from font size dictionaries
    :return:
    """

    plot_title_font_size = font_size[size_key]['plot_title_font_size']
    plot_font_size = font_size[size_key]['plot_font_size']
    long_font_size = font_size[size_key]['bibiplot_long_font_size']
    marker_size_2d = marker_size[size_key]['2d']

    # special handling for mouse datasets
    if dataset in ('motor', 'visual'):
        # from bibiplot_motor.ipynb
        geneExpr = pd.read_csv(f'data/mouse_{dataset}_cortex/geneExp.csv', index_col=0)
        geneExpr = pd.DataFrame(geneExpr, dtype='int')
        count = scipy.sparse.csr_matrix(geneExpr)  #.T  # transpose to put cells in rows, genes in columns
        seqDepth = np.sum(count, axis=1)  # sum gene enrichments across columns
        seqDepth = np.array(seqDepth)  # convert from matrix to vector
        X = np.log10(count / seqDepth * np.median(seqDepth) + 1)  # normalize  (not clear that multiplying by the median adds much value)
        X = np.array(X)
        X = X - X.mean(axis=0)
        X = X / np.std(X, axis=0)

        labels_X = geneExpr.columns.tolist()  # gene names

        ephysY = pd.read_csv(f'data/mouse_{dataset}_cortex/efeature.csv', index_col=0)
        ephysY.rename(columns=short_ephys_labels, inplace=True)
        # if type(ephysY.iloc[0, 0]) == str:
        #     # drop the first column if it contains strings (i.e., presumably cell names)
        #     ephysY.drop(columns=ephysY.columns[0], inplace=True)
        Y = ephysY.to_numpy() - np.mean(ephysY.to_numpy(), axis=0)
        Y = Y / np.std(Y, axis=0)

        labels_Y = ephysY.columns.tolist()

    else:  # User-uploaded data
        if dataset == 'morph':
            data_2.rename(columns=short_morph_labels, inplace=True)

        labels_X = data_1.columns.tolist()
        labels_Y = data_2.columns.tolist()

        # preprocess using user-selected method
        X = preprocess(data_1, preprocess_1)
        Y = preprocess(data_2, preprocess_2)

    Zx = d1[:, (x_col, y_col)]
    Zx = Zx / np.std(Zx, axis=0)

    Zy = d2[:, (x_col, y_col)]
    Zy = Zy / np.std(Zy, axis=0)

    Rho_x = np.corrcoef(np.concatenate((Zx, X), axis=1), rowvar=False)[2:, :2]
    Rho_y = np.corrcoef(np.concatenate((Zy, Y), axis=1), rowvar=False)[2:, :2]

    titles = (label_1, label_2)

    # Set font size for titles without affecting other annotations
    titles = [f'{t}' for t in titles]

    fig = make_subplots(rows=2, cols=2,
                        row_heights=[0, 1],
                        vertical_spacing=0.1,
                        horizontal_spacing=0.02,
                        column_titles=titles,
                        shared_xaxes=True,
                        shared_yaxes=True)

    unit = 1  # radius of circle

    # Select only the columns that will be plotted
    d1 = d1[:, [x_col, y_col]]
    d2 = d2[:, [x_col, y_col]]

    # Scale to be contained within the unit circle
    for i in range(2):
        d1[:, i] = d1[:, i] / np.max(np.abs(d1[:, i])) * 0.707
        d2[:, i] = d2[:, i] / np.max(np.abs(d2[:, i])) * 0.707

    marker_opacity = 0.6

    if color is None:
        # No color vector, so use a single group, single color, with no legend
        colors = 'black'
        use_categorical_colors = False
        show_legend = False
    else:

        categories = list(set(color))
        if len(categories) <= 10:  # arbitrary limit, but corresponds to length of color vector
            # category plot, show legend
            categories = sorted(categories)
            color_dict = {category: px.colors.qualitative.Plotly[i] for i, category in enumerate(categories)}
            use_categorical_colors = True
            show_legend = True
        else:
            # too many categories for legend.
            use_categorical_colors = False
            if color.dtype in ('string', 'object'):
                # plot black, no legend
                colors = 'black'
                show_legend = False
            else:
                # presumably numeric
                # continuous plot, show legend
                categories = sorted(categories)
                colors = color
                show_legend = True

    # Scatter plots
    for r in [2]:
        if use_categorical_colors:
            for category in categories:
                subset = list(color == category)
                c = 1
                fig.add_trace(go.Scatter(x=d1[subset, 0], y=d1[subset, 1], mode='markers',
                                         marker={'size': marker_size_2d, 'color': color_dict[category]},
                                         xaxis=f'x{plot_id(r, c)}',
                                         yaxis=f'y{plot_id(r, c)}',
                                         showlegend=False,
                                         hoverinfo="none",
                                         name=category,
                                         opacity=marker_opacity
                                         ),
                              row=r, col=c)
                c = 2
                fig.add_trace(go.Scatter(x=d2[subset, 0], y=d2[subset, 1], mode='markers',
                                         marker={'size': marker_size_2d, 'color': color_dict[category]},
                                         xaxis=f'x{plot_id(r, c)}',
                                         yaxis=f'y{plot_id(r, c)}',
                                         showlegend=True,
                                         hoverinfo="none",
                                         name=category,
                                         opacity=marker_opacity
                                         ),
                              row=r, col=c)
        else:
            # Create a single plot, using a color vector instead of a single color
            c = 1
            fig.add_trace(go.Scatter(x=d1[:, 0], y=d1[:, 1], mode='markers',
                                     marker={'size': marker_size_2d, 'color': colors, 'colorscale': 'Jet'},
                                     xaxis=f'x{plot_id(r, c)}',
                                     yaxis=f'y{plot_id(r, c)}',
                                     showlegend=False,
                                     hoverinfo="none",
                                     opacity=marker_opacity
                                     ),
                          row=r, col=c)
            c = 2
            fig.add_trace(go.Scatter(x=d2[:, 0], y=d2[:, 1], mode='markers',
                                     marker={'size': marker_size_2d, 'color': colors, 'colorscale': 'Jet', 'showscale': show_legend},
                                     xaxis=f'x{plot_id(r, c)}',
                                     yaxis=f'y{plot_id(r, c)}',
                                     showlegend=False, #show_legend,
                                     hoverinfo="none",
                                     opacity=marker_opacity
                                     ),
                          row=r, col=c)

        # Circles
        for c in [0, 1]:
            fig.add_shape(type='circle',
                          line={'color': 'Black', 'width': 1},
                          #line_color="Black",
                          xref='x', yref='y',
                          x0=-unit, y0=-unit, x1=unit, y1=unit,
                          row=r, col=c)

        # Lines (correlations between genes/features and the latent space)
        label_y_offset = 8  # pixels

        for c, (Z, F, labels, Rho) in enumerate(zip([Zx, Zy], [X, Y], [labels_X, labels_Y], [Rho_x, Rho_y])):

            # select top num_features as ranked by correlation with the latent space

            correlations = [np.sqrt(np.sum(Rho[i, :]**2)) for i in range(F.shape[1])]
            top_n_idx = np.argsort(correlations)[-num_features:]

            # find optimal radial coordinates of the top n feature annotations to limit overlap
            # (x, y, text, radius, angle, offset)
            points = [{'x': Rho[i, 0], 'y': Rho[i, 1], 'label': labels[i], 'r': np.sqrt(Rho[i, 0]**2 + Rho[i, 1]**2),
                       'theta': np.arctan2(Rho[i, 1], Rho[i, 0]) % twopi, 'shift': 0} for i in top_n_idx]
            deg = np.pi / 180  # 1 degree
            points = sorted(points, key=itemgetter('theta'))
            # now iterate in steps: first find shifts, then apply them
            n_iterations = 30
            force_neighborhood = 8 * deg  # was 10
            step = 2 * deg  # was 5

            # print('\t'.join([p['label'] for p in sorted(points, key=itemgetter('label'))]))

            for _ in range(n_iterations):
                # print('\t'.join([str(p['theta']) for p in sorted(points, key=itemgetter('label'))]))
                # find shifts
                for idx in range(len(points)):
                    if idx == 0:
                        force_below = (points[0]['theta'] - points[-1]['theta']) % twopi < force_neighborhood
                        force_above = (points[1]['theta'] - points[0]['theta']) % twopi < force_neighborhood
                    elif idx == len(points) - 1:
                        force_below = (points[idx]['theta'] - points[idx-1]['theta']) % twopi < force_neighborhood
                        force_above = (points[0]['theta'] - points[idx]['theta']) % twopi < force_neighborhood
                    else:
                        force_below = (points[idx]['theta'] - points[idx-1]['theta']) % twopi < force_neighborhood
                        force_above = (points[idx+1]['theta'] - points[idx]['theta']) % twopi < force_neighborhood

                    if force_below and force_above:
                        pass
                    elif force_above:
                        points[idx]['shift'] = -1 * step
                    elif force_below:
                        points[idx]['shift'] = step

                # apply shifts
                for p in points:
                    p['theta'] = (p['theta'] + p['shift']) % twopi
                    p['shift'] = 0

                # resort
                points = sorted(points, key=itemgetter('theta'))

            # for _ in range(n_iterations):
            #     # find shifts
            #     for idx in range(len(points)):
            #         force_below = idx > 0 and \
            #                       (points[idx]['theta'] - points[idx-1]['theta'] < force_neighborhood)
            #         force_above = idx < len(points) - 1 and \
            #                       (points[idx + 1]['theta'] - points[idx]['theta'] < force_neighborhood)
            #
            #         if force_below and force_above:
            #             pass
            #         elif force_above:
            #             points[idx]['shift'] = -1 * step
            #         elif force_below:
            #             points[idx]['shift'] = step
            #
            #     # apply shifts
            #     for p in points:
            #         p['theta'] += p['shift']
            #         p['shift'] = 0

            labels = np.array(labels)
            sig_idx = []
            # for i in range(F.shape[1]):
            #     if np.sqrt(np.sum(Rho[i,:]**2)) > .6:
            for i in top_n_idx:
                sig_idx.append(i)
                fig.add_shape(type='line',
                              xref='x', yref='y',
                              row=r, col=c + 1,
                              x0=0, y0=0,
                              x1=Rho[i, 0], y1=Rho[i, 1],
                              line={'color': 'gray', 'width': 1})

                if Rho[i, 0] < -0.2:
                    annotation_xanchor = 'left'
                elif Rho[i, 0] > 0.2:
                    annotation_xanchor = 'right'
                else:
                    annotation_xanchor = 'center'

                # fig.add_annotation(x=Rho[i, 0],
                #                    y=Rho[i, 1],
                #                    yshift=label_y_offset if Rho[i, 1] > -0.1 else -label_y_offset,
                #                    xanchor=annotation_xanchor,
                #                    text=labels[i],
                #                    showarrow=False,
                #                    row=r, col=c + 1)

                # alternate annotation format:
                # 1. rotate text so that it is radial
                # 2. perturb position (tangentially!) to avoid overlap
            for p in points:
                x = p['r'] * np.cos(p['theta'])
                y = p['r'] * np.sin(p['theta'])
                x = 0.6 * np.cos(p['theta'])
                y = 0.6 * np.sin(p['theta'])

                textangle = -np.arctan(np.tan(p['theta'])) * 180 / np.pi

                #textangle = -np.arctan(Rho[i, 1] / Rho[i, 0]) * 180 / np.pi
                fig.add_annotation(x=x,
                                   y=y,
                                   yshift=label_y_offset if y > -0.1 else -label_y_offset,
                                   xanchor='center', #annotation_xanchor,
                                   text=p['label'],
                                   textangle=textangle,
                                   showarrow=False,
                                   row=r, col=c + 1,
                                   font_size=plot_font_size if len(p['label']) < 14 else long_font_size)

            # Add labels to lines
            if False:
                fig.add_trace(go.Scatter(
                    x=Rho[sig_idx, 0], y=Rho[sig_idx, 1],
                    text=np.array(labels)[sig_idx],
                    mode='text',
                    showlegend=False,
                    hovertemplate='%{text}<extra></extra>'
                ),
                row=r, col=c + 1)

        # Axis labels, etc
        c = 1
        fig.update_yaxes(title_text=f'Component {y_col + 1}', row=r, col=c,
                         scaleanchor=f'x{plot_id(r, c)}', scaleratio=1,
                         showticklabels=False, showgrid=False, showline=False, visible=True, zeroline=False)
        c = 2
        fig.update_yaxes(scaleanchor=f'x{plot_id(r, c)}', scaleratio=1, row=r, col=c,
                         showticklabels=False, showgrid=False, showline=False, visible=True, zeroline=False)

        for c in [1, 2]:
            fig.update_xaxes(title_text=f'Component {x_col + 1}', row=r, col=c,
                             showticklabels=False, showgrid=False, showline=False, visible=True, zeroline=False)
            fig.update_xaxes(row=r, col=c,
                             showticklabels=False, showgrid=False, showline=False, visible=True, zeroline=False)

    # This appears to link x and y axes so both plots have same level of pan/zoom
    fig.update_xaxes(matches='x')
    fig.update_yaxes(matches='y')

    fig.update_layout(title_text='Top Feature Correlation with Latent Space',
                      plot_bgcolor='white',
                      font_size=plot_font_size,
                      title_font_size=plot_title_font_size,
                      title_yanchor='bottom',
                      title_pad={'b': plot_title_font_size * 1.5},
                      margin={'t': 100 + plot_title_font_size * 1.5})
    #fig.update_annotations(font_size=plot_font_size)

    if show_legend:
        fig.update_layout(legend={'itemsizing': 'constant', 'title': color_types_title.get(color.name, color.name)})

    return fig

def create_bibiplot2x2(d1, d2):
    """
    1. plot component 1 against component 2 and component 3 for modality 1 and modality 2
    2. color by cluster (or t-type) eventually
    3. plot lines for genes or electrophys features where length is correlation with latent space (max val 1.0)

    :return:
    """

    titles = ('Transcriptomic space', 'Electrophysiological space')

    fig = make_subplots(rows=2, cols=2,
                        column_titles=titles)

    unit = 1  # radius of circle

    d1 = d1[:, :3]
    d2 = d2[:, :3]

    for i in range(3):
        d1[:, i] = d1[:, i] / np.max(np.abs(d1[:, i])) * 0.707
        d2[:, i] = d2[:, i] / np.max(np.abs(d2[:, i])) * 0.707

    # Scatter plots
    for r in [1, 2]:
        c = 1
        fig.add_trace(go.Scatter(x=d1[:, 0], y=d1[:, r], mode='markers',
                                 marker={'size': 2},
                                 xaxis=f'x{plot_id(r, c)}',
                                 yaxis=f'y{plot_id(r, c)}',
                                 showlegend=False),
                      row=r, col=c)
        c = 2
        fig.add_trace(go.Scatter(x=d2[:, 0], y=d2[:, r], mode='markers',
                                 marker={'size': 2},
                                 xaxis=f'x{plot_id(r, c)}',
                                 yaxis=f'y{plot_id(r, c)}',
                                 showlegend=False),
                      row=r, col=c)

    for r in [0, 1]:
        for c in [0, 1]:
            fig.add_shape(type='circle',
                          line={'color': 'Black', 'width': 1},
                          #line_color="Black",
                          xref='x', yref='y',
                          x0=-unit, y0=-unit, x1=unit, y1=unit,
                          row=r, col=c)

    for r in [1, 2]:
        c = 1
        fig.update_yaxes(title_text=f'Component {r+1}', row=r, col=c,
                         scaleanchor=f'x{plot_id(r, c)}', scaleratio=1,
                         showticklabels=False)
        c = 2
        fig.update_yaxes(scaleanchor=f'x{plot_id(r, c)}', scaleratio=1, row=r, col=c,
                         showticklabels=False)
    for c in [1, 2]:
        fig.update_xaxes(title_text=f'Component 1', row=2, col=c,
                         showticklabels=False)
        fig.update_xaxes(row=1, col=c,
                         showticklabels=False)

    return fig


import glob
import logging
import logging.handlers
import os
import stat
import subprocess
from pathlib import Path

import pyudev

from lava_common.constants import DISPATCHER_DOWNLOAD_DIR as JOBS_DIR
from lava_common.exceptions import InfrastructureError
from lava_common.yaml import yaml_safe_dump, yaml_safe_load
from lava_dispatcher_host.docker_devices import Device, DeviceFilter

context = pyudev.Context()

logger = logging.getLogger("lava-dispatcher-host")
logger.addHandler(logging.handlers.SysLogHandler(address="/dev/log"))
logger.setLevel(logging.INFO)

def get_mapping_path(job_id):
    return os.path.join(JOBS_DIR, job_id, "usbmap.yaml")

def add_device_container_mapping(job_id, device_info, container, container_type="lxc"):
    validate_device_info(device_info)
    item = {
        "device_info": device_info,
        "container": container,
        "container_type": container_type,
        "job_id": job_id,
    }
    mapping_path = get_mapping_path(job_id)
    data = load_mapping_data(mapping_path)

    # remove old mappings for the same device_info
    newdata = [old for old in data if old["device_info"] != item["device_info"]]
    newdata.append(item)

    os.makedirs(os.path.dirname(mapping_path), exist_ok=True)
    with open(mapping_path, "w") as f:
        f.write(yaml_safe_dump(newdata))

def remove_device_container_mappings(job_id):
    os.unlink(get_mapping_path(job_id))

def validate_device_info(device_info):
    if not device_info:
        raise ValueError("Addind mapping for empty device info: %r" % device_info)
    if not any(device_info.values()):
        raise ValueError(
            "Addind mapping for device info with empty keys: %r" % device_info
        )

def share_device_with_container(options):
    data, job_id = find_mapping(options)
    if not data:
        return
    container = data["container"]
    device = options.device
    if not device.startswith("/dev/"):
        device = "/dev/" + device
    if not os.path.exists(device):
        logger.warning(f"Can't share {device}: file not found")
        return

    container_type = data["container_type"]
    if container_type == "lxc":
        share_device_with_container_lxc(container, device, job_id=job_id)
    elif container_type == "docker":
        share_device_with_container_docker(container, device, job_id=job_id)
    else:
        raise InfrastructureError('Unsupported container type: "%s"' % container_type)

def find_mapping(options):
    for mapping in glob.glob(get_mapping_path("*")):
        data = load_mapping_data(mapping)
        for item in data:
            if match_mapping(item["device_info"], options):
                job_id = str(Path(mapping).parent.name)
                return item, job_id
    return None, None

def load_mapping_data(filename):
    try:
        with open(filename) as f:
            data = yaml_safe_load(f) or []
        if isinstance(data, dict):
            data = [data]
        return data
    except FileNotFoundError:
        return []

def match_mapping(device_info, options):
    matched = False
    for k, v in device_info.items():
        if v:
            if k in options and getattr(options, k) == v:
                matched = True
            else:
                return False
        else:
            matched = True
    return matched

def log_sharing_device(device, container_type, container):
    logger.info(f"Sharing {device} with {container_type} container {container}")

def share_device_with_container_lxc(container, node, job_id):
    device = pyudev.Devices.from_device_file(context, node)
    pass_device_into_container_lxc(container, node, device.device_links, job_id)
    for child in device.children:
        if child.device_node:
            pass_device_into_container_lxc(
                child.device_node, child.device_links, job_id
            )

def pass_device_into_container_lxc(container, node, links=[], job_id=None):
    try:
        nodeinfo = os.stat(node)
        uid = nodeinfo.st_uid
        gid = nodeinfo.st_gid
        mode = "%o" % (0o777 & nodeinfo.st_mode)
    except FileNotFoundError as exc:
        logger.warning(
            f"Cannot share {node} with lxc container {container}: {exc.filename} not found"
        )
        return
    subprocess.check_call(["lxc-device", "-n", container, "add", node])
    log_sharing_device(node, "lxc", container)

    set_perms = f"chown {uid}:{gid} {node} && chmod {mode} {node}"
    subprocess.check_call(["lxc-attach", "-n", container, "--", "sh", "-c", set_perms])

    for link in links:
        create_link = f"mkdir -p {os.path.dirname(link)} && ln -f -s {node} {link}"
        subprocess.check_call(
            ["lxc-attach", "-n", container, "--", "sh", "-c", create_link]
        )

def pass_device_into_container_docker(
    container, container_id, node, links=[], job_id=None
):
    try:
        nodeinfo = os.stat(node)
        major = os.major(nodeinfo.st_rdev)
        minor = os.minor(nodeinfo.st_rdev)
        nodetype = "b" if stat.S_ISBLK(nodeinfo.st_mode) else "c"

        state = Path(JOBS_DIR) / job_id / (container_id + ".devices")
        device_filter = DeviceFilter(container_id, state)
        device_filter.add(Device(major, minor))
        device_filter.apply()
        device_filter.save(state)

    except FileNotFoundError as exc:
        logger.warning(
            f"Cannot share {node} with docker container {container}: {exc.filename} not found"
        )
        return

    # it's ok to fail; container might have already exited at this point.
    nodedir = os.path.dirname(node)
    uid = nodeinfo.st_uid
    gid = nodeinfo.st_gid
    mode = "%o" % (0o777 & nodeinfo.st_mode)
    subprocess.call(
        [
            "docker",
            "exec",
            container,
            "sh",
            "-c",
            f"mkdir -p {nodedir} && mknod {node} {nodetype} {major} {minor} && chown {uid}:{gid} {node} && chmod {mode} {node}",
        ]
    )

    for link in links:
        subprocess.call(
            [
                "docker",
                "exec",
                container,
                "sh",
                "-c",
                f"mkdir -p {os.path.dirname(link)} && ln -f -s {node} {link}",
            ]
        )

def share_device_with_container_docker(container, node, job_id=None):
    log_sharing_device(node, "docker", container)
    try:
        container_id = subprocess.check_output(
            ["docker", "inspect", "--format={{.ID}}", container], text=True
        ).strip()
    except subprocess.CalledProcessError:
        logger.warning(
            f"Cannot share {node} with docker container {container}: container not found"
        )
        return

    device = pyudev.Devices.from_device_file(context, node)
    pass_device_into_container_docker(
        container, container_id, node, device.device_links, job_id
    )

    for child in device.children:
        if child.device_node:
            pass_device_into_container_docker(
                container, container_id, child.device_node, child.device_links, job_id
            )


from datetime import datetime

from django.shortcuts import render, get_object_or_404, redirect

from categories.models import Category
from comments.forms import CommentForm
from comments.models import PostComment
from core.views import get_posts
from posts.models import Post

def index(request):
    head_post = Post.objects.filter(status='published').filter(published__lte=datetime.now()).last()
    head_post_categories = head_post.category.all()
    all_categories = Category.objects.all()

    posts = Post.objects.filter(status='published').filter(published__lte=datetime.now()).order_by('-published')[1:]

    ctx = get_posts(request, queryset=posts)
    return render(request, 'posts/index.html',
                  {'head_post': head_post, 'head_post_categories': head_post_categories,
                   'all_categories': all_categories, **ctx})

def post_detail(request, year, month, day, slug):
    post = get_object_or_404(Post, slug=slug, status='published', published__year=year,
                             published__month=month, published__day=day)
    post_categories = post.category.all()
    post_comments = post.comments.filter(parent__isnull=True).order_by('-created')

    if request.method == 'POST':
        comment_form = CommentForm(request.POST)
        if comment_form.is_valid():
            parent_obj = None

            try:
                parent_id = int(request.POST.get('parent_id'))
            except:
                parent_id = None

            if parent_id:
                parent_obj = PostComment.objects.get(id=parent_id)
                if parent_obj:
                    replay_comment = comment_form.save(commit=False)
                    replay_comment.parent = parent_obj

            new_comment = comment_form.save(commit=False)
            new_comment.post = post
            new_comment.save()
            return redirect(request.path)
    else:
        comment_form = CommentForm()

    try:
        previous_post = post.get_previous_by_published()
    except post.DoesNotExist:
        previous_post = post

    try:
        next_post = post.get_next_by_published()
    except post.DoesNotExist:
        next_post = post

    return render(request, 'posts/post_detail.html', {'post': post, 'post_categories': post_categories,
                                                      'comment_form': comment_form, 'post_comments': post_comments,
                                                      'previous_post': previous_post, 'next_post': next_post})


from enum import Enum

class PortType(Enum):
    PORT_INPUT = 'input';
    PORT_OUTPUT= 'output';

    def code(self):
        if self == PortType.PORT_INPUT:
            return 0
        else:
            return 1

class PortName(str,Enum):
    IN0 = "in0"
    IN1 = "in1"
    OUT0 = "out0"
    OUT1 = "out1"
    OUT2 = "out2"

    def code(self):
        data = {
            PortName.IN0:0,
            PortName.IN1:1,
            PortName.OUT0:2,
            PortName.OUT1:3,
            PortName.OUT2:4,
        }
        return data[self]

    @staticmethod
    def from_code(v):
        data = [PortName.IN0, PortName.IN1, PortName.OUT0,
                PortName.OUT1, PortName.OUT2]
        return data[int(v)]

    def __lt__(self,other):
        return str(self) < str(other)

class BlockType(str,Enum):
    DAC = 'dac';
    ADC = 'adc';
    CHIP_INPUT = 'chip_input';
    CHIP_OUTPUT = 'chip_output';
    TILE_INPUT = "tile_input";
    TILE_OUTPUT = "tile_output";
    MULT = "mult";
    INTEG = "integ";
    FANOUT = "fanout";
    LUT = "lut";
    NONE = "<none>";

    def code(self):
        mapping = {
            BlockType.DAC: 0,
            BlockType.MULT: 5,
            BlockType.INTEG: 6,
            BlockType.FANOUT: 7,
            BlockType.LUT: 8,
            BlockType.ADC: 9
        }
        return mapping[self]

class CircCmdType(Enum):
    USE_DAC = 'use_dac';
    USE_MULT = 'use_mult';
    USE_FANOUT = 'use_fanout';
    USE_INTEG = 'use_integ';
    USE_LUT = 'use_lut';
    USE_ADC = 'use_adc';
    DISABLE_DAC = 'disable_dac';
    DISABLE_MULT = 'disable_mult';
    DISABLE_INTEG = 'disable_integ';
    DISABLE_FANOUT = 'disable_fanout';
    DISABLE_LUT = 'disable_lut';
    DISABLE_ADC = 'disable_adc';
    CONNECT = 'connect';
    BREAK = 'break';
    CALIBRATE = 'calibrate';
    GET_INTEG_STATUS = 'get_integ_status';
    GET_ADC_STATUS = 'get_adc_status';
    WRITE_LUT = "write_lut";
    GET_STATE = "get_state";
    SET_STATE = "set_state";
    MEASURE = "measure";
    TUNE = "tune";
    DEFAULTS = "defaults";
    PROFILE = "profile";

class ExpCmdType(Enum):
    RESET = 'reset';
    USE_ANALOG_CHIP = 'use_analog_chip';
    SET_SIM_TIME= 'set_sim_time';
    USE_OSC = 'use_osc';
    RUN = 'run';

class CmdType(Enum):
    CIRC_CMD = 'circ_cmd';
    EXPERIMENT_CMD = 'exp_cmd';
    FLUSH_CMD = 'flush_cmd';


import sys
from collections import deque

INPUT = sys.stdin.readline
DIRECTIONS = ((0, 1), (0, -1), (1, 0), (-1, 0))

def bfs() -> str | int:
    while QUEUE_FIRES:
        x, y = QUEUE_FIRES.popleft()

        for dx, dy in DIRECTIONS:
            nx, ny = x + dx, y + dy

            if 0 <= nx < C and 0 <= ny < R:
                if not VISIT_F[ny][nx] and MAZE[ny][nx] != "#":
                    VISIT_F[ny][nx] = VISIT_F[y][x] + 1
                    QUEUE_FIRES.append((nx, ny))

    while QUEUE_JIHOON:
        x, y = QUEUE_JIHOON.popleft()

        for dx, dy in DIRECTIONS:
            nx, ny = x + dx, y + dy

            if 0 <= nx < C and 0 <= ny < R:
                if MAZE[ny][nx] != "#" and not VISIT_J[ny][nx]:
                    if not VISIT_F[ny][nx] or VISIT_F[ny][nx] > VISIT_J[y][x] + 1:
                        VISIT_J[ny][nx] = VISIT_J[y][x] + 1
                        QUEUE_JIHOON.append((nx, ny))
            else:
                return VISIT_J[y][x]

    return "IMPOSSIBLE"

if __name__ == "__main__":
    R, C = map(int, INPUT().split())
    QUEUE_JIHOON, QUEUE_FIRES, MAZE = deque(), deque(), []
    VISIT_J = [[0 for _ in range(C)] for __ in range(R)]
    VISIT_F = [[0 for _ in range(C)] for __ in range(R)]
    for Y in range(R):
        tmp = list(INPUT().rstrip().strip())
        for X in range(C):
            if tmp[X] == "J":
                QUEUE_JIHOON.append((X, Y))
                VISIT_J[Y][X] = 1
            elif tmp[X] == "F":
                QUEUE_FIRES.append((X, Y))
                VISIT_F[Y][X] = 1
        MAZE.append(tmp)
    print(bfs())


from django.conf import settings
from django.contrib import messages
from django.shortcuts import render, get_object_or_404, redirect
from django.core.mail import send_mail, EmailMultiAlternatives, EmailMessage
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.template.loader import get_template, render_to_string
from django.utils.html import strip_tags

from email.mime.image import MIMEImage

from jobs.models import JobListing
from companies.models import CompanyListing
from consultants.models import ConsultantListing
from groups.models import GroupListing
from events.models import EventListing
from django.db.models import Value, CharField, Q

from .models import NewsletterUser, Newsletter
from .forms import NewsletterUserSignUpForm, NewsletterCreationForm

# Create a Newsletter
def control_newsletter(request):
	jobs = JobListing.objects.all().order_by('-date_posted').annotate(type=Value('job', CharField()))
	companies = CompanyListing.objects.all().order_by('-date_posted').annotate(type=Value('company', CharField()))
	events = EventListing.objects.all().order_by('-date_posted').annotate(type=Value('event', CharField()))
	groups = GroupListing.objects.all().order_by('-date_posted').annotate(type=Value('group', CharField()))
	consultants = ConsultantListing.objects.all().order_by('-date_posted').annotate(type=Value('consultant', CharField()))

	results = list(jobs) + list(events) + list(companies) + list(groups) + list(consultants)
	results = sorted(results, key=lambda obj: obj.date_posted, reverse=True)

	form = NewsletterCreationForm(request.POST or None)
	if form.is_valid():
		instance = form.save()
		newsletter = Newsletter.objects.get(id=instance.id)
		if newsletter.status == "Send":
			subject, from_email, body = newsletter.subject, settings.EMAIL_HOST_USER, newsletter.body
			for newsletteruser in newsletter.email.all():
				mail = EmailMultiAlternatives(subject=subject,to=[newsletteruser.email], from_email=from_email)
				mail.attach_alternative(render_to_string('newsletters/newsletter_template.html', {'all_items_feed': results}), "text/html")
				mail.send()
			messages.success(request, 'The newsletter has been created and sent!', 'alert alert-success alert-dismissable')
			return redirect('control_newsletter_list')
		elif newsletter.status == "Draft":
			messages.success(request, 'The newsletter draft has been saved!', 'alert alert-success alert-dismissable')
			return redirect('control_newsletter_list')
	context = {
		"form": form,
	}
	template = 'control_panel/control_newsletter.html'
	return render(request, template, context)

# Edit a Newsletter
def control_newsletter_edit(request, pk):
	jobs = JobListing.objects.all().order_by('-date_posted').annotate(type=Value('job', CharField()))
	companies = CompanyListing.objects.all().order_by('-date_posted').annotate(type=Value('company', CharField()))
	events = EventListing.objects.all().order_by('-date_posted').annotate(type=Value('event', CharField()))
	groups = GroupListing.objects.all().order_by('-date_posted').annotate(type=Value('group', CharField()))
	consultants = ConsultantListing.objects.all().order_by('-date_posted').annotate(type=Value('consultant', CharField()))

	results = list(jobs) + list(events) + list(companies) + list(groups) + list(consultants)
	results = sorted(results, key=lambda obj: obj.date_posted, reverse=True)

	newsletter = get_object_or_404(Newsletter, pk=pk)

	if request.method == "POST":
		form = NewsletterCreationForm(request.POST, instance=newsletter)
		if form.is_valid():
			newsletter = form.save()
			if newsletter.status == 'Send':
				subject, from_email, body = newsletter.subject, settings.EMAIL_HOST_USER, newsletter.body
				for newsletteruser in newsletter.email.all():
					mail = EmailMultiAlternatives(subject=subject,to=[newsletteruser.email], from_email=from_email)
					mail.attach_alternative(render_to_string('newsletters/newsletter_template.html', {'all_items_feed': results}), "text/html")
					mail.send()
				messages.success(request, 'The newsletter has been sent!', 'alert alert-success alert-dismissable')
				return redirect('control_newsletter_list')
			elif newsletter.status == "Draft":
				messages.success(request, 'The newsletter draft has been saved!', 'alert alert-success alert-dismissable')
				return redirect('control_newsletter_list')
	else:
		form = NewsletterCreationForm(instance=newsletter)

	context = {
		"form": form,
	}

	template = 'control_panel/control_newsletter.html'
	return render(request, template, context)

# Newsletter Sign Up
def newsletter_signup(request):
	s_form = NewsletterUserSignUpForm(request.POST or None)
	if s_form.is_valid():
		instance = s_form.save(commit=False)
		if NewsletterUser.objects.filter(email=instance.email).exists():
			messages.warning(request, 'Your email is already signed up to the newsletter', 'alert alert-warning alert-dismissable')
		else:
			instance.save()
			messages.success(request, 'You have been subscribed to the newsletter', 'alert alert-success alert-dismissable')
			subject = "Tech Palmy Newsletter"
			from_email = settings.EMAIL_HOST_USER
			to_email = [instance.email]
			message = EmailMultiAlternatives(subject=subject, from_email=from_email, to=to_email)
			html_template = get_template("newsletters/signup_email.html").render()
			message.attach_alternative(html_template, "text/html")
			message.send()
			return redirect('sites-home')

	context = {
		's_form': s_form,
	}
	template = 'newsletters/signup.html'
	return render(request, template, context)

# Newsletter Unsubscribe
def newsletter_unsubscribe(request):
	u_form = NewsletterUserSignUpForm(request.POST or None)
	if u_form.is_valid():
		instance = u_form.save(commit=False)
		if NewsletterUser.objects.filter(email=instance.email).exists():
			NewsletterUser.objects.filter(email=instance.email).delete()
			messages.success(request, 'You have successfully unsubscribed!', 'alert alert-success alert-dismissable')
			subject = "Tech Palmy Newsletter"
			from_email = settings.EMAIL_HOST_USER
			to_email = [instance.email]
			message = EmailMultiAlternatives(subject=subject, from_email=from_email, to=to_email)
			html_template = get_template("newsletters/unsubscribe_email.html").render()
			message.attach_alternative(html_template, "text/html")
			message.send()
			return redirect('sites-home')
		else:
			messages.warning(request, 'Your email is not curently subsrcibed!', 'alert alert-warning alert-dismissable')
	context = {
		'u_form': u_form,
	}
	template = 'newsletters/unsubscribe.html'
	return render(request, template, context)

# Display ALL Newsletters
def control_newsletter_list(request):
	newsletters = Newsletter.objects.all().order_by('-created')
	paginator = Paginator(newsletters, 5)
	page = (request.GET.get('page'))
	try:
		items = paginator.page(page)
	except PageNotAnInteger:
		items = paginator.page(1)
	except EmptyPage:
		items = paginator.page(paginator.num_pages)

	index = items.number - 1
	max_index = len(paginator.page_range)
	start_index = index - 5 if index >= max_index -5 else 0
	end_index = index + 5 if index <= max_index -5 else max_index
	page_range = paginator.page_range[start_index:end_index]

	context = {
		"items": items,
		"page_range": page_range
	}
	template = "control_panel/control_newsletter_list.html"
	return render(request,template, context)

# Single Newsletter Detail
def control_newsletter_detail(request, pk):
	newsletter = get_object_or_404(Newsletter, pk=pk)

	context = {
	"newsletter": newsletter,
	}
	template = "control_panel/control_newsletter_detail.html"
	return render(request, template, context)

# Delete a Newsletter
def control_newsletter_delete(request, pk):
	newsletter = get_object_or_404(Newsletter, pk=pk)

	if request.method == "POST":
		form = NewsletterCreationForm(request.POST, instance=newsletter)
		if form.is_valid():
			newsletter.delete()
			messages.success(request, 'The newsletter has been deleted!', 'alert alert-success alert-dismissable')
			return redirect('control_newsletter_list')

	else:
		form = NewsletterCreationForm(instance=newsletter)

	context = {
		"form": form,
	}
	template = 'control_panel/control_newsletter_delete.html'
	return render(request, template, context)


from util import capture_results
from base import BaseTest
from ipaplatform.paths import paths
from ipahealthcheck.core import config, constants
from ipahealthcheck.ipa.plugin import registry
from ipahealthcheck.ipa.certs import IPACertmongerExpirationCheck
from unittest.mock import Mock
from mock_certmonger import create_mock_dbus, _certmonger
from mock_certmonger import get_expected_requests, set_requests

from datetime import datetime, timedelta, timezone

class TestExpiration(BaseTest):
    patches = {
        'ipahealthcheck.ipa.certs.get_expected_requests':
        Mock(return_value=get_expected_requests()),
        'ipalib.install.certmonger._cm_dbus_object':
        Mock(side_effect=create_mock_dbus),
        'ipalib.install.certmonger._certmonger':
        Mock(return_value=_certmonger())
    }

    def test_expiration(self):
        set_requests()

        framework = object()
        registry.initialize(framework)
        f = IPACertmongerExpirationCheck(registry)

        f.config = config.Config()
        f.config.cert_expiration_days = 7
        self.results = capture_results(f)

        assert len(self.results) == 2

        result = self.results.results[0]
        assert result.result == constants.ERROR
        assert result.source == 'ipahealthcheck.ipa.certs'
        assert result.check == 'IPACertmongerExpirationCheck'
        assert result.kw.get('key') == '1234'
        assert result.kw.get('msg') == 'Request id 1234 expired on ' \
                                       '19700101001704Z'

        result = self.results.results[1]
        assert result.result == constants.SUCCESS
        assert result.source == 'ipahealthcheck.ipa.certs'
        assert result.check == 'IPACertmongerExpirationCheck'
        assert result.kw.get('key') == '5678'

    def test_expiration_warning(self):
        warning = datetime.now(timezone.utc) + timedelta(days=20)
        replaceme = {
            'nickname': '7777',
            'cert-file': paths.RA_AGENT_PEM,
            'key-file': paths.RA_AGENT_KEY,
            'ca-name': 'dogtag-ipa-ca-renew-agent',
            'not-valid-after': int(warning.timestamp()),
        }

        set_requests(remove=0, add=replaceme)

        framework = object()
        registry.initialize(framework)
        f = IPACertmongerExpirationCheck(registry)

        f.config = config.Config()
        f.config.cert_expiration_days = 30
        self.results = capture_results(f)

        assert len(self.results) == 2

        result = self.results.results[0]
        assert result.result == constants.SUCCESS
        assert result.source == 'ipahealthcheck.ipa.certs'
        assert result.check == 'IPACertmongerExpirationCheck'
        assert result.kw.get('key') == '5678'

        result = self.results.results[1]
        assert result.result == constants.WARNING
        assert result.source == 'ipahealthcheck.ipa.certs'
        assert result.check == 'IPACertmongerExpirationCheck'
        assert result.kw.get('key') == '7777'
        assert 'expires in 19 days' in result.kw.get('msg')


"""Alter engagement alter banner_url

Revision ID: d0f92ae9ba77
Revises: 2545d45bb29c
Create Date: 2022-06-27 13:36:48.323235

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'd0f92ae9ba77'
down_revision = '2545d45bb29c'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('engagement', sa.Column('banner_filename', sa.String(), nullable=True))
    op.drop_column('engagement', 'banner_url')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('engagement', sa.Column('banner_url', sa.VARCHAR(), autoincrement=False, nullable=True))
    op.drop_column('engagement', 'banner_filename')
    # ### end Alembic commands ###


from conans import ConanFile, Meson, tools
import os
import shutil

class LibnameConan(ConanFile):
    name = "libdrm"
    description = "User space library for accessing the Direct Rendering Manager, on operating systems that support the ioctl interface"
    topics = ("conan", "libdrm", "graphics")
    url = "https://github.com/bincrafters/conan-libdrm"
    homepage = "https://gitlab.freedesktop.org/mesa/drm"
    license = "MIT"

    settings = "os", "arch", "compiler", "build_type"
    options = {
        "shared": [True, False],
        "fPIC": [True, False],
        "libkms": [True, False],
        "intel": [True, False],
        "radeon": [True, False],
        "amdgpu": [True, False],
        "nouveau": [True, False],
        "vmwgfx": [True, False],
        "omap": [True, False],
        "exynos": [True, False],
        "freedreno": [True, False],
        "tegra": [True, False],
        "vc4": [True, False],
        "etnaviv": [True, False],
        "valgrind": [True, False],
        "freedreno-kgsl": [True, False],
        "udev": [True, False]
    }
    default_options = {
        "shared": True,
        "fPIC": True,
        "libkms": True,
        "intel": True,
        "radeon": True,
        "amdgpu": True,
        "nouveau": True,
        "vmwgfx": True,
        "omap": False,
        "exynos": False,
        "freedreno": True,
        "tegra": False,
        "vc4": True,
        "etnaviv": False,
        "valgrind": False,
        "freedreno-kgsl": False,
        "udev": False
    }

    _source_subfolder = "source_subfolder"
    _build_subfolder = "build_subfolder"

    def build_requirements(self):
        if not tools.which("meson"):
            self.build_requires("meson/0.59.3")
        if not tools.which('pkg-config'):
            self.build_requires('pkgconf/1.7.4')

    def requirements(self):
        if self.options.intel:
            self.requires("libpciaccess/0.16@bincrafters/stable")

    def config_options(self):
        if self.settings.os == 'Windows':
            del self.options.fPIC

    def configure(self):
        del self.settings.compiler.libcxx
        del self.settings.compiler.cppstd

    def source(self):
        tools.get(**self.conan_data["sources"][self.version])
        extracted_dir = self.name + "-" + self.version
        os.rename(extracted_dir, self._source_subfolder)

    def _configure_meson(self):
        meson = Meson(self)

        defs={
            "cairo-tests" : "false",
            "install-test-programs": "false"
        }
        for o in ["libkms", "intel", "radeon", "amdgpu","nouveau", "vmwgfx", "omap", "exynos",
                  "freedreno", "tegra", "vc4", "etnaviv", "valgrind", "freedreno-kgsl", "udev"]:
            defs[o] = "true" if getattr(self.options, o) else "false"

        meson.configure(
            defs = defs,
            source_folder=self._source_subfolder,
            build_folder=self._build_subfolder)
        return meson

    def build(self):
        def _get_pc_files(package):
            if package in self.deps_cpp_info.deps:
                lib_path = self.deps_cpp_info[package].rootpath
                for dirpath, _, filenames in os.walk(lib_path):
                    for filename in filenames:
                        if filename.endswith('.pc'):
                            shutil.copyfile(os.path.join(dirpath, filename), filename)
                            tools.replace_prefix_in_pc_file(filename, lib_path)
                for dep in self.deps_cpp_info[package].public_deps:
                    _get_pc_files(dep)
        _get_pc_files('libpciaccess')
        meson = self._configure_meson()
        meson.build()

    def package(self):
        meson = self._configure_meson()
        meson.install()

    def package_info(self):
        self.cpp_info.includedirs.append(os.path.join('include', 'libdrm'))
        self.cpp_info.includedirs.append(os.path.join('include', 'libkms'))
        self.cpp_info.libs = tools.collect_libs(self)


import cx_Oracle
import os
from flask import Flask, request, jsonify
from modules.application import app
from config import oraDB

@app.route('/employer/<string:erni>',methods = ["GET"])
def get_emp(erni):
    try:
        res = None
        path = os.path.join(app.config['SCRIPT_FOLDER'],"validate_emp_nib#.sql")
        sql = open(path,"r")
        params = {"erni#":erni}
        with cx_Oracle.connect(f"{oraDB.user_name}/{oraDB.password}@{oraDB.db}") as conn:
            with conn.cursor() as cursor:
                results = cursor.execute(sql.read(),params)
                while True:
                    rows = results.fetchall()
                    if not rows:
                        break
                    for r in rows:
                        res = r[0]
                if res is None:
                    res = 'N'
                else:
                    res = 'Y'
        sql.close()
        return jsonify(success="Y",exists=res),200
    except Exception as e:
        return jsonify(success="N",message=f"System Error: {str(e)}"),500


import json
import copy
import sys
sys.path.append('/Library/Python/2.7/site-packages')
import requests
import geopy
from geopy.distance import vincenty

urlbiz = 'http://fyndo.herokuapp.com/api/selectBizInfos'
bizresp = requests.get(urlbiz)
biz = bizresp.json()

bdic = {}

for y in range(len(biz)):
    bdic[biz[y]['bizId']] = biz[y]['bizName']

urlfeed = 'http://fyndox.herokuapp.com/api/selectFeeds'
feedresp = requests.get(urlfeed)
feedsmain = feedresp.json()

urlupdate = 'http://fyndox.herokuapp.com/api/updateFeed'

def feedsupdate():
    x = input('Enter starting feeds id: ')
    y = input('Enter ending feed id: ')
    notupdatedfeeds = []
    feeds = []
    for i in range(len(feedsmain)):
        if int(feedsmain[i]['feedId']) >= int(x) and int(feedsmain[i]['feedId']) <= int(y):
            feeds.append(feedsmain[i])

    for i in range(len(feeds)):
        d = {}
        t=0
        results = {}

        if feeds[i]['shopName'] != "":

            for j in range(len(biz)):
                feedshop = []
                bizshop = []
                fscore = 0.0000
                bscore = 0.0000
                fdltlg = (feeds[i]['gpsLatitude'],feeds[i]['gpsLongitude'])
                bzltlg = (biz[j]['gpsLatitude'],biz[j]['gpsLongitude'])

                if feeds[i]['shopName'] is None:
                    feedshop = []
                else:
                    feedshop = feeds[i]['shopName'].lower().split(" ")

                if biz[j]['bizName'] is None:
                    bizshop = []
                else:
                    bizshop = biz[j]['bizName'].lower().split(" ")

                try:
                    feedshop.remove('')
                    feedshop.remove(' ')
                    bizshop.remove('')
                    bizshop.remove(' ')

                except:
                    mmmm = 0

                for k in range(len(feedshop)):
                    if feedshop[k] in bizshop:
                        fscore = fscore + 1

                for p in range(len(bizshop)):
                    if bizshop[p] in feedshop:
                        bscore = bscore + 1

                if len(feedshop) == 0:
                    fscore = 0.0000
                else:
                    fscore = fscore/int(len(feedshop))

                if len(bizshop) == 0:
                    bscore = 0.0000
                else:
                    bscore= bscore/int(len(bizshop))

                if fscore > 0.5 or bscore > 0.5 and int(vincenty(fdltlg,bzltlg).meters)<=25:
                    results[biz[j]['bizId']] = fscore + bscore

            if int(len(results)) >= 1:
                kh = sorted(results, key = results.__getitem__, reverse = True)
                print(feedshop)
                print(bdic[kh[0]])
                print('The biz is - ' + str(kh[0]) + ' total score is - ' + str(results[kh[0]]))
                d['feedId'] = feeds[i]['feedId']
                d['bizId'] = kh[0]
                d['userCatLevel1'] = feeds[i]['userCatLevel1']
                d['userCatLevel2'] = feeds[i]['userCatLevel2']
                d['title'] = feeds[i]['title']
                d['textContent'] = feeds[i]['textContent']
                d['sourceDevice'] = feeds[i]['sourceDevice']

                fdup = requests.post(urlupdate, data = json.dumps(d), headers = {'Content-type': 'application/json'})
                if fdup.status_code == 200:
                    print('Successfully updated - ' + str(feeds[i]['feedId']))
                    t=1
                else:
                    print(' Update unsuccessful - ' + str(feeds[i]['feedId']))
                    notupdatedfeeds.append(feeds[i])
        else:
            notupdatedfeeds.append(feeds[i])

        if t == 0:
            notupdatedfeeds.append(feeds[i])

    if int(len(notupdatedfeeds)) >= 1:
        print('Feeds not updated are - \n')
        for i in range(len(notupdatedfeeds)):
            print(str(notupdatedfeeds[i]['feedId']))


import random

from django.core.management import BaseCommand

from cabinet.constants.constants import Target
from cabinet.models import WorkRule
from clients.models import Bank
from settings.configs.banks import BankCode

class Command(BaseCommand):
    help = 'Тестовые данные для WorkRule'

    def handle(self, *args, **options):
        for bank, code in BankCode.CHOICES:
            if bank in ['rus_micro_finance', 'simple_finance']:
                continue
            print(bank)
            for target, name in Target.CHOICES:
                WorkRule.objects.create(
                    bank=Bank.objects.filter(code=bank).first(),
                    text='$TEST$',
                    bg_type=target,
                    commission=float(random.randint(1, 10)),
                    commission_on_excess=float(random.randint(1, 40)),
                )