# -*- coding: utf-8 -*-
"""
@author:XuMing(xuming624@qq.com)
@description:
"""

from typing import Any, List, Tuple

import torch
import torch.nn as nn

# constants:
CHECKPOINT_FILE = "checkpoint.torch"
CPU_DEVICE = torch.device("cpu")
GPU_DEVICE = torch.device("cuda")

def is_pos_int(number: int) -> bool:
    """
    Returns True if a number is a positive integer.
    """
    return type(number) == int and number >= 0

def is_pos_float(number: float) -> bool:
    """
    Returns True if a number is a positive float.
    """
    return type(number) == float and number >= 0.0

def is_pos_int_list(l: List) -> bool:
    """
    Returns True if a list contains positive integers
    """
    return type(l) == list and all(is_pos_int(n) for n in l)

def is_pos_int_tuple(t: Tuple) -> bool:
    """
    Returns True if a tuple contains positive integers
    """
    return type(t) == tuple and all(is_pos_int(n) for n in t)

def is_long_tensor(tensor: torch.Tensor) -> bool:
    """
    Returns True if a tensor is a long tensor.
    """
    if torch.is_tensor(tensor):
        return tensor.type().endswith("LongTensor")
    else:
        return False

def is_float_tensor(tensor: torch.Tensor) -> bool:
    """
    Returns True if a tensor is a float tensor.
    """
    if torch.is_tensor(tensor):
        return tensor.type().endswith("FloatTensor")
    else:
        return False

def is_double_tensor(tensor: torch.Tensor) -> bool:
    """
    Returns True if a tensor is a double tensor.
    """
    if torch.is_tensor(tensor):
        return tensor.type().endswith("DoubleTensor")
    else:
        return False

def is_leaf(module: nn.Module) -> bool:
    """
    Returns True if module is leaf in the graph.
    """
    assert isinstance(module, nn.Module), "module should be nn.Module"
    return len([c for c in module.children()]) == 0 or hasattr(module, "_mask")

def is_on_gpu(model: torch.nn.Module) -> bool:
    """
    Returns True if all parameters of a model live on the GPU.
    """
    assert isinstance(model, torch.nn.Module)
    on_gpu = True
    has_params = False
    for param in model.parameters():
        has_params = True
        if not param.data.is_cuda:
            on_gpu = False
    return has_params and on_gpu

def is_not_none(sample: Any) -> bool:
    """
    Returns True if sample is not None and constituents are not none.
    """
    if sample is None:
        return False

    if isinstance(sample, (list, tuple)):
        if any(s is None for s in sample):
            return False

    if isinstance(sample, dict):
        if any(s is None for s in sample.values()):
            return False
    return True


# import library for computer vision
import cv2

# READ, WRITE, AND DISPLAY AN IMAGE

# read an image
image = cv2.imread(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\image_flower.jpg')
# storing image reading into a variable to use it multiple times without rereading
# less expensive method

# showing the image
cv2.imshow('The title of the window for the image of flower, 9th June 2021', image)

# put execution on hold, value 0 allows forever until window is closed
cv2.waitKey(0)

# close the window to show next image
cv2.destroyAllWindows()

# write the image
cv2.imwrite(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\cv outputs\image_flower.png',
            image)
# output is "True" for successful write as .png file at the location

# COLOR SPACE CONVERSION

# read an image
image2 = cv2.imread(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\Penguins.jpg')

# show the image
cv2.imshow(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\BGR_Penguins',
           image2)
cv2.waitKey(0)
cv2.destroyAllWindows()

# convert the BGR image into grayscale
image2 = cv2.cvtColor(image2,cv2.COLOR_BGR2GRAY)

# show the converted image
cv2.imshow(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\Gray_Penguins',
           image2)
cv2.waitKey(0)
cv2.destroyAllWindows()

# EDGE DETECTION

# read an image
image3 = cv2.imread(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\Penguins.jpg')

# detect the edges using Canny()
cv2.imwrite(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\cv outputs\edges_Penguins.jpg',
            cv2.Canny(image3,200,300))

# show the image edges
cv2.imshow(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\edges_Penguins',
           image3)
cv2.waitKey(0)
cv2.destroyAllWindows()

# FACE DETECTION

# import haarcascade_frontalface_default.xml classifier
face_detection = cv2.CascadeClassifier(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\haarcascades\haarcascades\haarcascade_frontalface_default.xml')

# read an image
image4 = cv2.imread(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\AB.jpg')

# convert the BGR image to only acceptable grayscale format
gray = cv2.cvtColor(image4, cv2.COLOR_BGR2GRAY)

# perform face detection
faces = face_detection.detectMultiScale(gray, 1.3, 5)

# draw a rectangle to indicate the face
for (x,y,w,h) in faces:
    image4 = cv2.rectangle(image4,(x,y),(x+w, y+h),(0,255,0),3)
    # (0,255,0) indicates the color of the rectangle
    # 3 indicates the thickness of the border lines

# write the image
cv2.imwrite(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\cv outputs\Face_AB.jpg',
            image4)

# EYE DETECTION

# import the haarcascade_eye.xml classifier
eye_cascade = cv2.CascadeClassifier(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\haarcascades\haarcascades\haarcascade_eye.xml')

# read an image
image5 = cv2.imread(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\AB_Eye.jpg')

# convert the BGR image to only acceptable grayscale format
gray2 = cv2.cvtColor(image5, cv2.COLOR_BGR2GRAY)

# perform eye detection
eyes = eye_cascade.detectMultiScale(gray2, 1.03, 5)

# draw a rectangle to indicate the eyes
for (ex,ey,ew,eh) in eyes:
    image5 = cv2.rectangle(image5,(ex,ey),(ex+ew, ey+eh),(0,0,255),2)

# write  the image
cv2.imwrite(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\cv outputs\Eye_AB.jpg',
            image5)


from PyQt5 import QtCore, QtGui, QtWidgets

class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.background = QtWidgets.QLabel(self.centralwidget)
        self.background.setGeometry(QtCore.QRect(-170, -50, 2291, 1191))
        self.background.setText("")
        self.background.setPixmap(QtGui.QPixmap("../Images/bg.png"))
        self.background.setScaledContents(True)
        self.background.setObjectName("background")
        self.label_1 = QtWidgets.QLabel(self.centralwidget)
        self.label_1.setGeometry(QtCore.QRect(20, 10, 1891, 1021))
        self.label_1.setText("")
        self.label_1.setPixmap(QtGui.QPixmap("../Images/background_2.png"))
        self.label_1.setScaledContents(True)
        self.label_1.setObjectName("label_1")
        self.label_wa = QtWidgets.QLabel(self.centralwidget)
        self.label_wa.setGeometry(QtCore.QRect(180, 320, 441, 161))
        self.label_wa.setText("")
        self.label_wa.setPixmap(QtGui.QPixmap("../Images/new_tab.png"))
        self.label_wa.setScaledContents(True)
        self.label_wa.setObjectName("label_wa")
        self.label_mc = QtWidgets.QLabel(self.centralwidget)
        self.label_mc.setGeometry(QtCore.QRect(180, 710, 441, 161))
        self.label_mc.setText("")
        self.label_mc.setPixmap(QtGui.QPixmap("../Images/new_tab.png"))
        self.label_mc.setScaledContents(True)
        self.label_mc.setObjectName("label_mc")
        self.label_ig = QtWidgets.QLabel(self.centralwidget)
        self.label_ig.setGeometry(QtCore.QRect(180, 510, 441, 161))
        self.label_ig.setText("")
        self.label_ig.setPixmap(QtGui.QPixmap("../Images/new_tab.png"))
        self.label_ig.setScaledContents(True)
        self.label_ig.setObjectName("label_ig")
        self.label_yt = QtWidgets.QLabel(self.centralwidget)
        self.label_yt.setGeometry(QtCore.QRect(180, 110, 441, 161))
        self.label_yt.setText("")
        self.label_yt.setPixmap(QtGui.QPixmap("../Images/new_tab.png"))
        self.label_yt.setScaledContents(True)
        self.label_yt.setObjectName("label_yt")
        self.Gif_1 = QtWidgets.QLabel(self.centralwidget)
        self.Gif_1.setGeometry(QtCore.QRect(760, 270, 720, 405))
        self.Gif_1.setText("")
        self.Gif_1.setPixmap(QtGui.QPixmap("../Gifs/center.gif"))
        self.Gif_1.setScaledContents(True)
        self.Gif_1.setObjectName("Gif_1")
        self.Gif_4 = QtWidgets.QLabel(self.centralwidget)
        self.Gif_4.setGeometry(QtCore.QRect(850, 700, 571, 141))
        self.Gif_4.setText("")
        self.Gif_4.setPixmap(QtGui.QPixmap("../Gifs/listening.gif"))
        self.Gif_4.setScaledContents(True)
        self.Gif_4.setObjectName("Gif_4")
        self.Gif_3 = QtWidgets.QLabel(self.centralwidget)
        self.Gif_3.setGeometry(QtCore.QRect(920, 60, 431, 201))
        self.Gif_3.setText("")
        self.Gif_3.setPixmap(QtGui.QPixmap("../Gifs/INITILIZING.gif"))
        self.Gif_3.setScaledContents(True)
        self.Gif_3.setObjectName("Gif_3")
        self.Gif_2 = QtWidgets.QLabel(self.centralwidget)
        self.Gif_2.setGeometry(QtCore.QRect(1500, 110, 351, 261))
        self.Gif_2.setText("")
        self.Gif_2.setPixmap(QtGui.QPixmap("../Gifs/round_1.gif"))
        self.Gif_2.setScaledContents(True)
        self.Gif_2.setObjectName("Gif_2")
        self.label_9 = QtWidgets.QLabel(self.centralwidget)
        self.label_9.setGeometry(QtCore.QRect(1500, 560, 261, 121))
        self.label_9.setText("")
        self.label_9.setPixmap(QtGui.QPixmap("../Images/Start.png"))
        self.label_9.setScaledContents(True)
        self.label_9.setObjectName("label_9")
        self.label_10 = QtWidgets.QLabel(self.centralwidget)
        self.label_10.setGeometry(QtCore.QRect(1500, 700, 261, 111))
        self.label_10.setText("")
        self.label_10.setPixmap(QtGui.QPixmap("../Images/quiet.png"))
        self.label_10.setScaledContents(True)
        self.label_10.setObjectName("label_10")
        self.date_time_day_temp = QtWidgets.QTextBrowser(self.centralwidget)
        self.date_time_day_temp.setGeometry(QtCore.QRect(230, 890, 521, 81))
        self.date_time_day_temp.setStyleSheet("background-color: Transparent;\n"
"color: rgb(85, 255, 255);\n"
"border-color: rgb(0, 0, 255);")
        self.date_time_day_temp.setObjectName("date_time_day_temp")
        self.pushButton_start = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_start.setGeometry(QtCore.QRect(1510, 580, 241, 81))
        self.pushButton_start.setStyleSheet("background-color: Transparent;")
        self.pushButton_start.setText("")
        self.pushButton_start.setObjectName("pushButton_start")
        self.pushButton_quit = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_quit.setGeometry(QtCore.QRect(1510, 710, 241, 81))
        self.pushButton_quit.setStyleSheet("background-color: Transparent;")
        self.pushButton_quit.setText("")
        self.pushButton_quit.setObjectName("pushButton_quit")
        self.pushButton_youtube = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_youtube.setGeometry(QtCore.QRect(240, 140, 331, 91))
        self.pushButton_youtube.setStyleSheet("background-color: Transparent;\n"
"font: 75 28pt \"Arial\";\n"
"color: rgb(0, 255, 255);")
        self.pushButton_youtube.setObjectName("pushButton_youtube")
        self.pushButton_whatapp = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_whatapp.setGeometry(QtCore.QRect(240, 350, 331, 91))
        self.pushButton_whatapp.setStyleSheet("background-color: Transparent;\n"
"font: 75 28pt \"Arial\";\n"
"color: rgb(0, 255, 255);")
        self.pushButton_whatapp.setObjectName("pushButton_whatapp")
        self.pushButton_instagram = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_instagram.setGeometry(QtCore.QRect(230, 540, 331, 91))
        self.pushButton_instagram.setStyleSheet("background-color: Transparent;\n"
"font: 75 28pt \"Arial\";\n"
"color: rgb(0, 255, 255);")
        self.pushButton_instagram.setObjectName("pushButton_instagram")
        self.pushButton_myclass = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_myclass.setGeometry(QtCore.QRect(240, 740, 331, 91))
        self.pushButton_myclass.setStyleSheet("background-color: Transparent;\n"
"font: 75 28pt \"Arial\";\n"
"color: rgb(0, 255, 255);")
        self.pushButton_myclass.setObjectName("pushButton_myclass")
        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 1922, 21))
        self.menubar.setObjectName("menubar")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "MainWindow"))
        self.date_time_day_temp.setHtml(_translate("MainWindow", "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n"
"<meta name=\"qrichtext\" content=\"1\" />\n"
"

"))
        self.pushButton_youtube.setText(_translate("MainWindow", "YOU TUBE"))
        self.pushButton_whatapp.setText(_translate("MainWindow", "WHAT\'S APP"))
        self.pushButton_instagram.setText(_translate("MainWindow", "INSTAGRAM"))
        self.pushButton_myclass.setText(_translate("MainWindow", "MY CLASS"))

if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = Ui_MainWindow()
    ui.setupUi(MainWindow)
    MainWindow.show()
    MainWindow.showFullScreen()
    sys.exit(app.exec_())


"""
Contains operations regarding token model
"""

import uuid
from datetime import datetime, timedelta
from django.core.exceptions import ObjectDoesNotExist

from rest_framework.response import Response

from ..models import User,User_token
from .user import authenticate

def insert_token(*, request):
    """
    Inserts token in user_token table with row user_id.
    """

    auth = authenticate(request=request)
    if auth is not True:
        return auth

    token = str(uuid.uuid4())
    expiration_time= datetime.now() + timedelta(seconds=3600)
    user = User.objects.get(email=request.data.get('email'))

    try:
        user_token = User_token.objects.get(user=user)
        user_token.token=token
        user_token.expiration_time = expiration_time
        user_token.save()

    except ObjectDoesNotExist:
        User_token.objects.create(user=user, token=token, expiration_time=expiration_time)

    return Response(data={"token":token},status=200,content_type="application/json")

def validate_token(*, request):
    """
    Validates token and returns if user is admin
    """
    token= request.headers.get("Authorization")
    status=True

    error=''

    if not token:
        error+=  "Token is required"
        status = False

    if not isinstance(token , str):
        error = "Token should be str"
        status = False

    if not token.startswith("Bearer"):
        error= "Invalid Token"
        status = False

    try:
        token=token.split(' ')[1]

    except IndexError:
        error= "Invalid Token"
        status = False

    if status is False:
        return Response(data={"message":error},status=400)

    current_time = datetime.now().time()

    try:
        user_token= User_token.objects.get(token=token)

    except ObjectDoesNotExist:
        return None

    if user_token.expiration_time<current_time:
        return None

    user=User.objects.get(id=user_token.user_id)
    return user


"""Urls for the demo of Zinnia"""
from functools import partial

from django.conf import settings
from django.contrib import admin
from django.contrib.sitemaps.views import index
from django.contrib.sitemaps.views import sitemap
from django.urls import include, re_path
from django.views.defaults import bad_request, page_not_found, permission_denied, server_error
from django.views.generic.base import RedirectView
from django.views.static import serve
from django_xmlrpc.views import handle_xmlrpc

from zinnia.sitemaps import AuthorSitemap
from zinnia.sitemaps import CategorySitemap
from zinnia.sitemaps import EntrySitemap
from zinnia.sitemaps import TagSitemap

urlpatterns = [
    re_path(r'^$', RedirectView.as_view(url='/blog/', permanent=True)),
    re_path(r'^blog/', include('zinnia.urls')),
    re_path(r'^comments/', include('django_comments.urls')),
    re_path(r'^xmlrpc/$', handle_xmlrpc),
    re_path(r'^i18n/', include('django.conf.urls.i18n')),
    re_path(r'^admin/doc/', include('django.contrib.admindocs.urls')),
    re_path(r'^admin/', admin.site.urls),
]

sitemaps = {
    'tags': TagSitemap,
    'blog': EntrySitemap,
    'authors': AuthorSitemap,
    'categories': CategorySitemap
}

urlpatterns += [
    re_path(r'^sitemap.xml$',
            index,
            {'sitemaps': sitemaps}),
    re_path(r'^sitemap-(?P<section>.+)\.xml$',
            sitemap,
            {'sitemaps': sitemaps}),
]

urlpatterns += [
    re_path(r'^400/$', partial(bad_request, exception=None)),
    re_path(r'^403/$', partial(permission_denied, exception=None)),
    re_path(r'^404/$', partial(page_not_found, exception=None)),
    re_path(r'^500/$', server_error),
]

if settings.DEBUG:
    urlpatterns += [
        re_path(r'^media/(?P<path>.*)$', serve,
                {'document_root': settings.MEDIA_ROOT})
    ]


import importlib
import pkgutil
from argparse import ArgumentParser
from re import Pattern
from types import ModuleType
from typing import List, Union, cast
from unittest import mock

import pytest

def pytest_addoption(parser):
    """Standard pytest hook invoked to add options to pytest CLI"""
    parser.addoption(
        "--xfail-providers-with-missing-deps",
        action="store_true",
        help="Skip tests on providers with optional dependencies",
    )

def pytest_runtest_setup(item):
    """Standard pytest hook invoked before each test execution"""
    try:
        skip_providers_with_optdeps = getattr(
            item.config.option, "xfail_providers_with_missing_deps"
        )
    except AttributeError:
        pass
    else:
        if skip_providers_with_optdeps:
            from lexicon._private.discovery import find_providers

            providers = find_providers()
            skip = [
                available
                for provider, available in providers.items()
                if provider in item.parent.name.lower()
            ]
            if skip and not skip[0]:
                pytest.xfail(
                    "Test expected to fail because --skip-providers-with-missing-deps "
                    "is set and provider has missing required dependencies."
                )

@pytest.fixture(scope="session")
def mock_provider():
    """
    Create a fake provider module, and mock relevant
    functions to make it appear as a real module.
    """
    from lexicon.interfaces import Provider as BaseProvider

    class Provider(BaseProvider):
        """
        Fake provider to simulate the provider resolution from configuration,
        and to have execution traces when lexicon client is invoked
        """

        @staticmethod
        def get_nameservers() -> Union[List[str], List[Pattern]]:
            return cast(List[str], [])

        @staticmethod
        def configure_parser(parser: ArgumentParser) -> None:
            pass

        def authenticate(self):
            print("Authenticate action")

        def create_record(self, rtype, name, content):
            return {
                "action": "create",
                "domain": self.domain,
                "type": rtype,
                "name": name,
                "content": content,
            }

        def list_records(self, rtype=None, name=None, content=None):
            return {
                "action": "list",
                "domain": self.domain,
                "type": rtype,
                "name": name,
                "content": content,
            }

        def update_record(self, identifier=None, rtype=None, name=None, content=None):
            return {
                "action": "update",
                "domain": self.domain,
                "identifier": identifier,
                "type": rtype,
                "name": name,
                "content": content,
            }

        def delete_record(self, identifier=None, rtype=None, name=None, content=None):
            return {
                "action": "delete",
                "domain": self.domain,
                "identifier": identifier,
                "type": rtype,
                "name": name,
                "content": content,
            }

        def _request(self, action="GET", url="/", data=None, query_params=None):
            # Not use for tests
            pass

    original_iter = pkgutil.iter_modules
    original_import = importlib.import_module

    with mock.patch("lexicon._private.discovery.pkgutil.iter_modules") as mock_iter:
        with mock.patch(
            "lexicon._private.discovery.importlib.import_module"
        ) as mock_import:

            def return_iter(path):
                """
                This will include an adhoc fakeprovider module
                to the normal return of pkgutil.iter_modules.
                """
                modules = list(original_iter(path))
                modules.append((None, "fakeprovider", None))
                return modules

            mock_iter.side_effect = return_iter

            def return_import(module_name):
                """
                This will return a adhoc fakeprovider module if necessary,
                or fallback to the normal return of importlib.import_module.
                """
                if module_name == "lexicon._private.providers.fakeprovider":
                    module = ModuleType("lexicon._private.providers.fakeprovider")
                    setattr(module, "Provider", Provider)
                    return module
                return original_import(module_name)

            mock_import.side_effect = return_import

            yield


import numpy as np
from collections import namedtuple, defaultdict, Counter
from itertools import product, combinations
import json
import argparse
import input_output
import os
import traceback

parser = argparse.ArgumentParser(description='Check that all families were phased.')
parser.add_argument('phase_dir', type=str, help='Directory with phase data.')
parser.add_argument('data_dir', type=str, default=None, help='Directory gen data.')
parser.add_argument('param_file', type=str, default=None, help='Param file.')
parser.add_argument('--family_size', type=int, default=None, help='Family size.')

args = parser.parse_args()

with open('%s/info.json' % args.phase_dir, 'r') as f:
	info = json.load(f)

ped_file = info['ped_file']

if args.data_dir is None:
	data_dir = info['data_dir']
else:
	data_dir = args.data_dir

if args.param_file is None:
	param_file = info['param_file']
else:
	param_file = args.param_file

# start by pulling families, only consider nuclear families
families = input_output.pull_families(ped_file)
families = [x for x in families if x.num_ancestors()==2 and len(x.ordered_couples)==1]

sample_file = '%s/samples.json' % data_dir
with open(sample_file, 'r') as f:
    sample_ids = set(json.load(f))

with open(param_file, 'r') as f:
    params = json.load(f)

for family in families:
    to_be_removed = [x for x in family.individuals if x not in sample_ids]
    family.prune(to_be_removed)

families = [x for x in families if x.num_descendents()>0]
print(len(families), 'have genomic data')

# if we have a family size limit, then enforce
if args.family_size is not None:
	families = [x for x in families if len(x)==args.family_size]
	print(len(families), 'of size', args.family_size)

for family in families:
    to_be_removed = [x for x in family.individuals if (x not in params and '%s.%s' % (family.id, x) not in params)]
    family.prune(to_be_removed)

families = [x for x in families if x.num_descendents()>0]
print(len(families), 'have parameters')

#pulls phase data from a file
def pull_phased_chroms(filename):
	with open(filename, 'r') as f:
		header = next(f).strip().split('\t')[1:-2] # skip header
		individuals = [x[:-4] for x in header if x.endswith('_mat')]

		chrs = set()
		for line in f:
			pieces = line.strip().split('\t', maxsplit=1)
			chrs.add(pieces[0][3:])

	return chrs, individuals

def write_to_file(outf, chrom, positions, states):
	# write final states to file
	change_indices = [-1] + np.where(np.any(states[1:, :]!=states[:-1, :], axis=1))[0].tolist() + [positions.shape[0]-1]
	for j in range(1, len(change_indices)):
		start_index, end_index = change_indices[j-1]+1, change_indices[j]
		outf.write('%s\t%s\t%d\t%d\n' % (
						'chr' + chrom,
						'\t'.join(map(str, states[start_index, :])),
						positions[start_index, 0], positions[end_index, 1]))

# writes a recomb only file
def create_recomb_file(filename):
	states = []
	positions = []
	current_chrom = None
	with open(filename, 'r') as f, open(filename + '.recomb.txt', 'w+') as outf:
		header = next(f).strip().split('\t')[1:-2] # skip header

		for line in f:
			pieces = line.strip().split('\t')
			chrom = pieces[0][3:]
			if current_chrom is None:
				current_chrom = chrom
				positions.append([int(x) for x in pieces[-2:]])
				states.append([int(x) for x in pieces[1:-2]])
			elif chrom == current_chrom:
				positions.append([int(x) for x in pieces[-2:]])
				states.append([int(x) for x in pieces[1:-2]])
			else:
				# write data from previous chrom
				states = np.array(states)
				positions = np.array(positions)

				# ignore deletions and haplotypes, we only care about crossovers
				states[:, :4] = -1

				# if this is a hard to sequences region, we don't know the exact location of crossovers
				states[states[:, -1]==1, :] = -1

				# ignore hard to sequence regions as well
				#states[:, -1] = -1

				write_to_file(outf, current_chrom, positions, states)

				# reset for next chrom
				current_chrom = chrom
				positions = [[int(x) for x in pieces[-2:]]]
				states = [[int(x) for x in pieces[1:-2]]]

		# write data from last chrom
		states = np.array(states)
		positions = np.array(positions)

		# ignore deletions and haplotypes, we only care about crossovers
		states[:, :4] = -1

		# if this is a hard to sequences region, we don't know the exact location of crossovers
		states[states[:, -1]==1, :] = -1

		# ignore hard to sequence regions as well
		#states[:, -1] = -1

		write_to_file(outf, current_chrom, positions, states)

has_no_data = []
has_missing_chroms = []
has_missing_individuals = []
is_missing_phase_data = []
all_good = 0

for family in families:
	phase_filename = '%s/%s.phased.txt' % (args.phase_dir, family.id)
	if os.path.isfile(phase_filename):
		try:
			chroms, individuals = pull_phased_chroms(phase_filename)
			missing_chroms = set([str(x) for x in range(1, 23)]) - chroms
			if len(missing_chroms)>0:
				has_missing_chroms.append(family.id)
			elif len(individuals) != len(family.individuals):
				has_missing_individuals.append((family.id, set(family.individuals)-set(individuals)))
			else:
				all_good += 1
				#create_recomb_file(phase_filename)
		except StopIteration:
			has_no_data.append(family.id)
	else:
		is_missing_phase_data.append(family.id)

print('has no data', has_no_data)
print('has missing chroms', has_missing_chroms)
print('has missing individuals', has_missing_individuals)
print('is missing phase data', is_missing_phase_data)
print('all good', all_good)


# 689. 三个无重叠子数组的最大和
# 给你一个整数数组 nums 和一个整数 k ，找出三个长度为 k 、互不重叠、且 3 * k 项的和最大的子数组，并返回这三个子数组。

# 以下标的数组形式返回结果，数组中的每一项分别指示每个子数组的起始位置（下标从 0 开始）。如果有多个结果，返回字典序最小的一个。

# 来源：力扣（LeetCode）
# 链接：https://leetcode-cn.com/problems/maximum-sum-of-3-non-overlapping-subarrays
# 著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
from typing import List

class Solution:
    def maxSumOfThreeSubarrays(self, nums: List[int], k: int) -> List[int]:
        n = len(nums)
        preSum = [0]
        for i in nums:
            preSum.append(preSum[-1]+i)
        dp=[[0]*4 for _ in range(n+5)]
        for i in range(n-k+1,0,-1):
            for j in range(1,4):
                dp[i][j]=max(dp[i+1][j],dp[i+k][j-1]+preSum[i+k-1]-preSum[i-1])
        res = []
        i,j = 1,3
        while j>0:
            if dp[i+1][j]>dp[i+k][j-1]+preSum[i+k-1]-preSum[i-1]:
                i+=1
            else:
                res.append(i-1)
                i+=k
                j-=1
        return res

sol = Solution()
nums = [1,2,1,2,6,7,5,1]
k = 2
print(sol.maxSumOfThreeSubarrays(nums,k))


from django.contrib import admin
from django.urls import path, include
from account_main import views

urlpatterns = [
    path('signup/', views.signup, name='signup'),
    path('signin/', views.signin, name='signin'),
    path('set_user_data/', views.set_user_data, name="set_user_data"),
    path('login/', views.login, name="login"),
    path('logout/', views.LogoutView.as_view(), name="logout"),
    path('login1/', views.LoginView.as_view(), name="login1"),
]


import json
from box import Box
from .helpers import check_required_keys, fatal
from .images import (
    resample_ct_like_spect,
    resample_roi_like_spect,
    convert_ct_to_densities,
)
from .opendose import get_svalue_and_mass_scaling
from pathlib import Path
import os
import opengate as gate
from opengate import g4_units
from opengate.geometry.materials import HounsfieldUnit_to_material
from opengate.image import get_translation_between_images_center, read_image_info
import pkg_resources
import SimpleITK as itk
import numpy as np

def read_dose_rate_options(json_file):
    print(json_file)
    if json_file is None:
        options = init_dose_rate_options()
    else:
        f = open(json_file).read()
        options = Box(json.loads(f))
    check_dose_rate_options(options)
    return options

def check_dose_rate_options(options):
    ref = init_dose_rate_options()
    check_required_keys(options, ref.keys())

def init_dose_rate_options():
    options = Box()
    options.number_of_threads = 1
    options.density_tolerance_gcm3 = 0.1
    return options

def get_timepoint_output_folder(output_folder, cycle, timepoint, name="doserate"):
    folder = (
            Path(output_folder)
            / Path(cycle.cycle_id)
            / Path(timepoint.acquisition_id)
            / name
    )
    os.makedirs(folder, exist_ok=True)
    return folder

def simu_default_init(sim):
    sim.visu_type = "vrml"
    m = gate.g4_units.m
    world = sim.world
    world.size = [2 * m, 2 * m, 2 * m]

    # cuts
    sim.physics_manager.physics_list_name = "G4EmStandardPhysics_option3"
    sim.physics_manager.enable_decay = True
    sim.physics_manager.set_production_cut("world", "all", 1 * m)

    # add stat actor
    stats = sim.add_actor("SimulationStatisticsActor", "stats")
    stats.track_types_flag = True
    stats.output = "stats.txt"

def sim_add_waterbox(sim, ct_filename):
    wb = sim.add_volume("Box", "waterbox")
    info = read_image_info(ct_filename)
    wb.size = info.size
    wb.material = "G4_WATER"
    wb.color = [0, 0, 1, 1]
    return wb

def simu_add_ct(sim, ct_filename, density_tolerance_gcm3):
    if sim.visu:
        return sim_add_waterbox(sim, ct_filename)
    ct = sim.add_volume("Image", "ct")
    ct.image = ct_filename
    # material used by default
    ct.material = "G4_AIR"
    gcm3 = g4_units.g_cm3
    tol = density_tolerance_gcm3 * gcm3
    # default tables
    table_mat = pkg_resources.resource_filename(
        "rpt_dosi", "data/Schneider2000MaterialsTable.txt"
    )
    table_density = pkg_resources.resource_filename(
        "rpt_dosi", "data/Schneider2000DensitiesTable.txt"
    )
    ct.voxel_materials, materials = HounsfieldUnit_to_material(
        sim, tol, table_mat, table_density
    )
    print(f"Density tolerance = {tol / gcm3} gcm3")
    print(f"Number of materials in the CT : {len(ct.voxel_materials)} materials")
    # ct.dump_label_image = "labels.mhd"

    # production cut
    mm = gate.g4_units.mm
    sim.physics_manager.set_production_cut("ct", "all", 1 * mm)
    return ct

def simu_add_activity_source(
        sim,
        ct,
        activity_filename,
        rad,
):
    rad_list = {
        "Lu177": {"Z": 71, "A": 177, "name": "Lutetium 177"},
        "Y90": {"Z": 39, "A": 90, "name": "Yttrium 90"},
        "In111": {"Z": 49, "A": 111, "name": "Indium 111"},
        "I131": {"Z": 53, "A": 131, "name": "Iodine 131"},
    }

    # Activity source from an image
    source = sim.add_source("VoxelsSource", "vox")
    source.mother = ct.name
    source.particle = "ion"
    source.ion.Z = rad_list[rad]["Z"]
    source.ion.A = rad_list[rad]["A"]
    source.image = activity_filename
    source.direction.type = "iso"
    keV = g4_units.keV
    source.energy.mono = 0 * keV
    if ct.name == "ct":
        source.position.translation = get_translation_between_images_center(
            ct.image, activity_filename
        )
    return source

def add_dose_actor(sim, ct, source):
    # add dose actor (get the same size as the source)
    source_info = read_image_info(source.image)
    dose = sim.add_actor("DoseActor", "dose")
    dose.output = "edep.mhd"
    dose.mother = ct.name
    dose.size = source_info.size
    dose.spacing = source_info.spacing
    # translate the dose the same way as the source
    dose.translation = source.position.translation
    # set the origin of the dose like the source
    if not sim.user_info.visu:
        dose.img_coord_system = True
    dose.hit_type = "random"
    # dose.hit_type = "pre"
    dose.uncertainty = True
    dose.square = True
    dose.gray = True
    return dose

def scale_to_absorbed_dose_rate(
        activity,
        dose_in_gray,
        simu_activity,
        calibration_factor,
        verbose=True,
):
    dose_a = itk.GetArrayFromImage(dose_in_gray)
    activity_a = itk.GetArrayFromImage(activity)

    volume_voxel_mL = np.prod(activity.GetSpacing()) / 1000
    total_activity = np.sum(activity_a) * volume_voxel_mL / calibration_factor

    if verbose:
        print(f"Total activity in the image FOV: {total_activity / 1e6:.2f} MBq")

    print(f"dose mean = {np.mean(dose_a)} gray.s-1")
    dose_a = dose_a / simu_activity * total_activity
    print(f"dose mean after scaling = {np.mean(dose_a)} gray.s-1")

    # create output image
    o = itk.GetImageFromArray(dose_a)
    o.CopyInformation(dose_in_gray)
    return o

def spect_calibration(spect, calibration_factor, concentration_flag, verbose=True):
    # get voxel volume
    volume_voxel_mL = np.prod(spect.GetSpacing()) / 1000
    arr = itk.GetArrayFromImage(spect)
    total_activity = np.sum(arr) * volume_voxel_mL / calibration_factor
    if verbose:
        print(f"Total activity in the image FOV: {total_activity / 1e6:.2f} MBq")
    # calibration
    if concentration_flag:
        arr = arr * volume_voxel_mL / calibration_factor
    else:
        arr = arr / calibration_factor

    # create output image
    o = itk.GetImageFromArray(arr)
    o.CopyInformation(spect)
    return o

def dose_hanscheid2018(spect_Bq, roi, time_sec, svalue, mass_scaling):
    """
    Input image and ROI must be numpy arrays
    - spect must be in Bq (not concentration)
    - svalue in mGy/MBq/s
    - time in seconds
    - output is in Gray
    """
    # compute mean activity in the ROI, in MBq
    v = spect_Bq[roi == 1]
    At = np.sum(v) / 1e6

    # S is in (mGy/MBq/s), so we get dose in mGy
    dose = mass_scaling * At * svalue * (2 * time_sec) / np.log(2) / 1000

    return dose

def dose_hanscheid2017(spect_Bqml, roi, time_sec, pixel_volume_ml):
    """
    Input img and ROI must be numpy arrays

    """

    time_h = time_sec / 3600
    time_eff_h = 67.0

    # compute mean activity in the ROI
    v = spect_Bqml[roi == 1] / pixel_volume_ml
    Ct = np.mean(v) / 1e6

    #
    dose = 0.125 * Ct * np.power(2, time_h / time_eff_h) * time_eff_h

    return dose

def dose_hanscheid2018_from_filenames(
        spect_file, ct_file, roi_file, phantom, roi_name, rad_name, time_h
):
    # read spect
    spect = itk.ReadImage(spect_file)
    spect_a = itk.GetArrayFromImage(spect)

    # prepare ct : resample and densities
    ct = itk.ReadImage(ct_file)
    ct_a = resample_ct_like_spect(spect, ct)
    densities = convert_ct_to_densities(ct_a)
    volume_voxel_mL = np.prod(spect.GetSpacing()) / 1000

    # prepare roi : resample
    roi = itk.ReadImage(roi_file)
    roi_a = resample_roi_like_spect(spect, roi)

    # get svalues and scaling
    svalue, mass_scaling = get_svalue_and_mass_scaling(
        phantom, roi_a, roi_name, rad_name, volume_voxel_mL, densities
    )

    # compute dose
    time_sec = time_h * 3600
    dose = dose_hanscheid2018(spect_a, roi_a, time_sec, svalue, mass_scaling)
    print(f"Dose for {roi_file}: {dose:.2f} Gray")
    return dose

def get_roi_list(filename):
    # open the file
    with open(filename, "r") as f:
        data = json.load(f)
    l = []
    for item in data:
        l.append((item["roi_filename"], item["roi_name"]))
    return l

def fit_exp_linear(x, y):
    denegative = 1
    if y[0] < 0:
        denegative = -1
        y[0] = -y[0]
    y = np.log(y)
    k, A_log = np.polyfit(x, y, 1)
    A = np.exp(A_log) * denegative
    if k > 0:
        k = 0
    return A, k

def fit_triexpo(times, activities):
    t0 = float(times[0])
    t1 = float(times[1])
    t3 = float(times[2])
    d0 = float(activities[0])
    d1 = float(activities[1])
    d3 = float(activities[2])
    if d3 == d1 or d1 == d0 or d0 == d3:
        d0 += 0.0001
        d1 += 0.0002
        d3 += 0.0003
    d1lin = (((d3 - d0) / (t3 - t0)) * t1 + (d0 - ((d3 - d0) / ((t3 - t0)) * t0)))
    A3 = d3 / (np.exp(-0.001 * t3))
    d1_lowslope = A3 * np.exp(-0.001 * t1)
    if d1 < d1_lowslope:
        if d0 < d3:
            if d1 < d1lin:
                d1 = d1lin
        else:
            d1 = d1_lowslope
    if d3 < 0.1 * d1:
        d3 = 0.1 * d1
    if d0 < 0.2 * d1:
        d0 = 0.2 * d1
    if d3 > d1:
        k3 = -0.001
        A3 = d3 / (np.exp(k3 * t3))

        x = np.array([t1, t3])
        y = np.array([d1 - (A3 * np.exp(k3 * t1)), 0.01 * d3])
        A2, k2 = fit_exp_linear(x, y)
        if (A2 + A3) < 0:
            A2 = -A3
            A1 = 0
            k1 = 0
        else:
            A1 = -(A2 + A3)
            k1 = -1.3
    else:
        x = np.array([t1, t3])
        y = np.array([d1, d3])
        A3, k3 = fit_exp_linear(x, y)
        if k3 > -0.001:
            k3 = -0.001
            A3 = d3 / (np.exp(k3 * t3))
        x = np.array([t0, t1])
        y = np.array([d0 - (A3 * np.exp(k3 * t0)), 0.01 * d1])
        A2, k2 = fit_exp_linear(x, y)
        if (A2 + A3) < 0:
            A2 = -A3
            A1 = 0
            k1 = 0
        else:
            A1 = -(A2 + A3)
            k1 = -1.3
    params = np.array([A1, k1, A2, k2, A3, k3])
    return params


from flask import Flask, request

app = Flask(__name__)

stores = [
    {
        "name": "My Store",
        "items": [
            {
                "name": "Chair",
                "price": 15.99
            }
        ]
    },
    {
        "name": "PMGV Store",
        "items": [
            {
                "name": "PC",
                "price": 2515.99
            }
        ]
    }
]

@app.route("/store") # http://127.0.0.1:5000/store
def get_stores():
    return {"stores": stores}

@app.route("/store", methods=['POST'])
def create_store():
    new_store = { "name": "My Store from Home", "items":[] }
    stores.append( new_store )
    return new_store, 201


# validator.py
# Program validates the solution of a nondeterministic turing machine.
# Project: Turing-machine (FLP project 2)
# Author: Michal Šedý <xsedym02@stud.fit.vutbr.cz>
# Last change: 24.04.2022

import sys
from collections import defaultdict
from curses.ascii import islower, isupper

class Ntm:
    def __init__(self) -> None:
        self.state = "S"
        self.head = 0
        self.tape = list()
        self.rules = defaultdict(lambda: defaultdict(set))
        self.step_cnt = 1

    def load_rules(self, file_name: str):
        """Load Turing machine rules and the input tape from a given file.

        Args:
            file_name (str): file with a turing machine

        Raises:
            e: Bad file format
        """
        fd = open(file_name, "r")
        lines = fd.readlines()
        tape = "S" + lines[-1].rstrip()
        self.tape = list(tape)
        for line in lines[:-1]:
            line = line.rstrip()
            if not line:
                continue
            try:
                s1, l1, s2, l2 = line.split(" ")
            except Exception as e:
                print(tape)
                print(line.split(" "))
                raise e
            self.rules[s1][l1].add((s2, l2))

    def next_tape(self, tape):
        """Test that the Turing machine configuration has been reached correctly.

        Args:
            tape (list): Turing machine tape.
        """
        def cmp_tapes(t1, t2):
            if not all(map(lambda x: islower(x) or x == "#", t1)) or not all(map(lambda x: islower(x) or x == "#", t2)):
                return False
            return t1 == t2

        s1 = self.state
        l1 = self.tape[self.head+1]
        s2 = ""
        l2 = ""

        new_head = self.head
        test_tape1 = self.tape[:self.head] + self.tape[self.head+2:]

        if self.head != 0 and isupper(tape[self.head-1]):
            l2 = "L"
            s2 = tape[self.head-1]
            test_tape2 = tape[:self.head-1] + [tape[self.head]] + tape[self.head+2:]
            new_head -= 1
        elif isupper(tape[self.head]):
            l2 = tape[self.head+1]
            s2 = tape[self.head]
            test_tape2 = tape[:self.head] + tape[self.head+2:]
        elif isupper(tape[self.head+1]):
            l2 = "R"
            s2 = tape[self.head+1]
            if self.head+2 == len(self.tape):
                test_tape2 = tape[:-3]
            else:
                test_tape2 = tape[:self.head] + tape[self.head+2:]
            new_head += 1
        else:
            print("DIFF: State is not on the expected possition or on the tape.", file=sys.stderr)
            print(f"Mismatch between {self.step_cnt} and {self.step_cnt+1} step.", file=sys.stderr)
            print(f"{self.step_cnt}:{''.join(self.tape)}\n{self.step_cnt+1}:{''.join(tape)}", file=sys.stderr)
            return False

        if s1 not in self.rules or l1 not in self.rules[s1] or (s2, l2) not in self.rules[s1][l1]:
            print(f"DIFF: No rule '{s1} {l1} {s2} {l2}' for the tape modification.", file=sys.stderr)
            print(f"Mismatch between {self.step_cnt} and {self.step_cnt+1} step.", file=sys.stderr)
            print(f"{self.step_cnt}:{''.join(self.tape)}\n{self.step_cnt+1}:{''.join(tape)}", file=sys.stderr)
            return False

        if not cmp_tapes(test_tape1, test_tape2):
            print("DIFF: More states in the tape or tape modification withou head on.", file=sys.stderr)
            print(f"Mismatch between {self.step_cnt} and {self.step_cnt+1} step.", file=sys.stderr)
            print(f"{self.step_cnt}:{''.join(self.tape)}\n{self.step_cnt+1}:{''.join(tape)}", file=sys.stderr)
            return False

        self.tape = tape
        self.head = new_head
        self.state = self.tape[new_head]
        self.step_cnt += 1

        return True

class Validator:
    def __init__(self, input_file, rules_file) -> None:
        self.ntm = Ntm()
        self.ntm.load_rules(rules_file)
        self.cmpt_steps = list()
        with open(input_file, "r") as fd:
            for line in fd.readlines():
                self.cmpt_steps.append(list(line.rstrip()))

    def validate(self):
        if "S" not in self.cmpt_steps[0]:
            print("DIFF: No initial state S.", file=sys.stderr)
            print(f"Expected:{''.join(self.ntm.tape)}\nPROLOG:{''.join(self.cmpt_steps[0])}", file=sys.stderr)
            return False

        if "F" not in self.cmpt_steps[-1]:
            print("DIFF: No final state F.", file=sys.stderr)
            print(f"PROLOG:{''.join(self.cmpt_steps[-1])}", file=sys.stderr)
            return False

        if self.ntm.tape != self.cmpt_steps[0]:
            print("DIFF: Input tape differ in rules and output.", file=sys.stderr)
            print(f"ORIG:{''.join(self.ntm.tape)}\nPROLOG:{''.join(self.cmpt_steps[0])}", file=sys.stderr)
            return False

        for step in self.cmpt_steps[1:]:
            if not self.ntm.next_tape(step):
                return False

        return True

def main():
    if {"-h", "--help"}.intersection(sys.argv):
        print("USAGE:")
        print("validator.py test_input rules")
        sys.exit(0)

    if len(sys.argv) != 3:
        raise Exception("Bad number of program parameters.")

    validator = Validator(sys.argv[1], sys.argv[2])
    if not validator.validate():
        sys.exit(1)

if __name__ == "__main__":
    main()


import torch
import torchvision.utils as vutils

from core.abstract.abcvalidator import ABCValidator

class Validator(ABCValidator):
    def __init__(self, config, model, writer):
        super().__init__(config, model, writer)

        self.gen = model

        # Create batch of latent vectors that we will use to visualize the progression of the generator
        self.fixed_noise = torch.randn(self.config.num_visual_img, self.gen.num_latent_z, 1, 1,
                                       device=self.config.device)

    def step(self, epoch, step):
        results = {
            'ssid': 0,
            'ppim': 0,
        }

        self.gen.eval()

        with torch.no_grad():
            fake_img = self.gen(self.fixed_noise).detach()
            norm_fake_imgs = vutils.make_grid(fake_img, padding=2, normalize=True)

        self.writer.add_image('valid/fake_img', norm_fake_imgs, step)

        return results, 0


"""
Cleans OpenStack from test entities

Usage: openstack_clean [(-u | --unassigned)] [--keyword=<keyword>] [--read-only] [--verbose]

Options:
    -u --unassigned      Deletes not assigned to any tenant entities
    --keyword=<keyword>  Keyword for searching users and tenants [default: $test]
    --read-only          Just display objects which should be removed
    -v --verbose         Output debug messages

"""
import logbook as log

from utils import find_first, setup_backend_logbook
from os_interfaces.openstack_wrapper import openstack
from functools import lru_cache, reduce

@lru_cache()
def get_tenant(tenant_id):
    return openstack.get_tenant(tenant_id)

def ignore_exception(fn):
    def inner(*args, **kwargs):
        try:
            return fn(*args, **kwargs)
        except Exception:
            import traceback
            traceback.print_exc()
    return inner

def get_tenants_and_users(keyword):
    id_pairs = []
    all_tenants = openstack.get_tenants()
    tenants = [tenant.id for tenant in all_tenants if keyword in tenant.description]
    for tenant in tenants:
        users = openstack.client_keystone.tenants.list_users(tenant)
        user = users[0].id if users else None
        id_pairs.append((user, tenant))

    all_users = openstack.client_keystone.users.list()
    users = [(user.id, None) for user in all_users
             if keyword in user.name and find_first(id_pairs, lambda x: x[0] == user.id) is None]
    id_pairs.extend(users)

    return id_pairs

def accum_total(accum, resources):
    for resource in resources:
        if accum.get(resource, None):
            accum[resource] += resources[resource]
        else:
            accum[resource] = resources[resource]

    return accum

def delete_by_keyword(keyword, read_only):
    id_pairs = get_tenants_and_users(keyword)
    total_list = []
    for id_pair in id_pairs:
        try:
            deleted = openstack.final_delete(*id_pair)
            total_list.append(deleted)
        except Exception:
            log.warning("Deleting of tenant {} has failed", id_pair, exc_info=True)

    if total_list:
        total = reduce(accum_total, total_list, {})
        total_str = ",".join("%s %s" % (k, total[k]) for k in sorted(total))
        log.info('Deleted {}', total_str)
    else:
        log.info('Nothing to delete by keyword {}', keyword)

@ignore_exception
def delete_floatingips(read_only):
    all_ips = openstack.client_neutron.list_floatingips()['floatingips']
    ips = [ip['id'] for ip in all_ips if not get_tenant(ip['tenant_id'])]

    if ips:
        log.info("Delete unassigned ips: {}", ",".join(ips))
    if not read_only:
        for ip in ips:
            openstack.delete_floating_ip(ip)

    return len(ips)

@ignore_exception
def delete_instances(read_only):
    all_instances = openstack.get_nova_servers()
    instances = [instance for instance in all_instances if not get_tenant(instance.tenant_id)]

    for instance in instances:
        log.info('Deleting unassigned instance {} in tenant {}', instance.id, instance.tenant_id)
        if not read_only:
            openstack.client_nova.servers.delete(instance)

    return len(instances)

@ignore_exception
def delete_volumes(read_only):
    all_volumes = openstack.get_volumes()
    volumes = [volume for volume in all_volumes if not get_tenant(getattr(volume, 'os-vol-tenant-attr:tenant_id'))]
    for volume in volumes:
        log.info('Deleting unassigned volume {}, name: {}, size: {}, image_name: {}. tenant: {}',
                 volume.id, volume.name, volume.size, getattr(volume, "volume_image_metadata", None)["image_name"],
                 getattr(volume, 'os-vol-tenant-attr:tenant_id'))
        if not read_only:
            try:
                openstack.client_cinder.volumes.delete(volume.id)
            except Exception as e:
                log.warning("Can't remove volume {}: {}", volume._info, e)

    return len(volumes)

@ignore_exception
def delete_snapshots(read_only):
    all_snapshots = openstack.get_snapshots()
    snapshots = [snapshot for snapshot in all_snapshots if not get_tenant(snapshot.project_id)]
    for snapshot in snapshots:
        log.info('Deleting unassigned snapshot {}, tenant: {}', snapshot, snapshot.project_id)
        if not read_only:
            try:
                openstack.client_cinder.volume_snapshots.delete(snapshot.id)
            except Exception as e:
                log.warning("Can't remove snapshot {}: {}", snapshot._info, e)

    return len(snapshots)

@ignore_exception
def delete_images(read_only):
    all_images = openstack.get_nova_images()
    images = 0
    for image in all_images:
        tenant_id = image.metadata.get('owner_id', None)
        if tenant_id is not None and not get_tenant(tenant_id):
            log.info('Deleted unassigned image {}', image.id)
            if not read_only:
                openstack.client_nova.images.delete(image.id)
            images += 1

    return images

@ignore_exception
def delete_router(router, read_only):
    info = router["external_gateway_info"]
    ips = [ip["ip_address"] for ip in info["external_fixed_ips"] if ip]
    log.info('Deleting unassigned router {}, network {}, ip {}, tenant {}',
             router["id"], info["network_id"], ",".join(ips), router["tenant_id"])

    search_opts = {'device_owner': 'network:router_interface',
                   'device_id': router["id"]}
    ports = openstack.client_neutron.list_ports(**search_opts)['ports']
    for port in ports:
        log.info("Removes port {} from router", port["mac_address"])
        if not read_only:
            openstack.client_neutron.remove_interface_router(router["id"], body=dict(port_id=port['id']))
    if not read_only:
        openstack.client_neutron.delete_router(router["id"])

@ignore_exception
def delete_routers(read_only):
    all_routers = openstack.get_routers()
    routers = [router for router in all_routers if not get_tenant(router['tenant_id'])]
    for router in routers:
        delete_router(router, read_only)

    return len(routers)

@ignore_exception
def delete_networks(read_only):
    all_networks = openstack.get_networks()
    networks = [network for network in all_networks if not get_tenant(network['tenant_id'])]
    for network in networks:
        log.info('Deleted unassigned network {}, tenant {}', network["id"], network["tenant_id"])
        if not read_only:
            try:
                openstack.client_neutron.delete_network(network["id"])
            except Exception as e:
                log.warning("Can't delete network {}: {}", network, e)

    return len(networks)

@ignore_exception
def delete_subnets(read_only):
    all_subnets = openstack.get_subnets()
    subnets = [subnet for subnet in all_subnets if not get_tenant(subnet['tenant_id'])]
    for subnet in subnets:
        log.info('Deleted unassigned subnet {}, name: {}, gateway: {}, network: {}, tenant {}',
                 subnet["id"], subnet["name"], subnet["gateway_ip"], subnet["network_id"], subnet["tenant_id"])
        if not read_only:
            try:
                openstack.client_neutron.delete_subnet(subnet["id"])
            except Exception as e:
                log.warning("Can't delete subnet {}: {}", subnet, e)

    return len(subnets)

@ignore_exception
def delete_ports(read_only):
    all_ports = openstack.get_ports()
    ports = [port for port in all_ports if port['tenant_id'] and not get_tenant(port['tenant_id'])]
    for port in ports:
        log.info('Deleted unassigned port {}, ips: {}, network: {}, tenant: {}',
                 port["mac_address"], ",".join(ip["ip_address"] for ip in port["fixed_ips"]), port["network_id"], port["tenant_id"])
        if not read_only:
            try:
                openstack.client_neutron.delete_port(port['id'])
            except Exception as e:
                log.warning("Can't delete port {}: {}", port, e)

    return len(ports)

@ignore_exception
def delete_security_groups(read_only):
    all_groups = openstack.get_neutron_security_groups()
    groups = [group['id'] for group in all_groups if not get_tenant(group['tenant_id'])]
    for group in groups:
        log.info('Deleted unassigned security group {}', group)
        if not read_only:
            openstack.client_neutron.delete_security_group(group)

    return len(groups)

def delete_unassigned_resources(read_only):
    fips = delete_floatingips(read_only)
    instances = delete_instances(read_only)
    volumes = delete_volumes(read_only)
    snapshots = delete_snapshots(read_only)
    images = delete_images(read_only)
    routers = delete_routers(read_only)
    networks = delete_networks(read_only)
    subnets = delete_subnets(read_only)
    ports = delete_ports(read_only)
    sgroups = delete_security_groups(read_only)

    log.info('Deleted unassigned resources: {} floating ips, {} instances, {} volumes, {} snapshots, {} images, '
             '{} routers, {} networks, {} subnets, {} ports, {} security groups', fips, instances, volumes, snapshots,
             images, routers, networks, subnets, ports, sgroups)

def main():
    import docopt
    opt = docopt.docopt(__doc__)
    verbose = opt["--verbose"]

    with setup_backend_logbook('stderr', openstack_log_level="DEBUG" if verbose else "INFO",
                               min_level="DEBUG" if verbose else "INFO"):
        read_only = opt["--read-only"]
        if opt['--unassigned']:
            delete_unassigned_resources(read_only)
        delete_by_keyword(opt['--keyword'], read_only)

if __name__ == "__main__":
    main()


import os
import json
import time

import subprocess
import re
import time
import sys
import re
import select

pass_text = '''

        ██████╗  █████╗ ███████╗███████╗
        ██╔══██╗██╔══██╗██╔════╝██╔════╝
        ██████╔╝███████║███████╗███████╗
        ██╔═══╝ ██╔══██║╚════██║╚════██║
        ██║     ██║  ██║███████║███████║
        ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝
'''
fail_text = '''

        ███████╗ █████╗ ██╗██╗
        ██╔════╝██╔══██╗██║██║
        █████╗  ███████║██║██║
        ██╔══╝  ██╔══██║██║██║
        ██║     ██║  ██║██║███████╗
        ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝
'''

COMMAND_DIRECTORY = os.path.join(os.path.dirname(__file__), 'commands')

if not os.path.exists(COMMAND_DIRECTORY):
    os.mkdir(COMMAND_DIRECTORY)

def is_buffered_line(start_str, buffered):
    return start_str.startswith(buffered) or re.match(start_str, buffered)

def is_stuff_to_read(p):
    return bool(select.select([p.stdout, ], [], [], 0.0)[0])

def write_buffer(buffered):
    term_width = os.get_terminal_size().columns
    buffered = re.sub("OK", "\033[42m\033[30m{}\033[00m".format("OK".ljust(term_width)), buffered )
    buffered  = re.sub("FAILED(.*)\n", r"\033[41m\033[37m{}\1\033[00m\n".format("FAILED".ljust(term_width)), buffered )

    re_text = r'(?P<total_count>[\d]+) example[s]*, (?P<failure_count>[\d]+) failure[s]*'
    result = re.search(re_text, buffered)

    if result and int(result.group("failure_count")) > 0:
        total_match = result.group(0)
        buffered = re.sub(total_match, "\033[41m\033[37m{}\1\033[00m".format(total_match.ljust(term_width)), buffered)
    elif result and int(result.group("failure_count")) == 0:
        total_match = result.group(0)
        buffered = re.sub(total_match, "\033[42m\033[30m{}\033[00m".format(total_match.ljust(term_width)), buffered)

    sys.stdout.write(buffered)

def run_command(c, directory, conn):
    term_width = os.get_terminal_size().columns

    print("running command: " + c)
    print("in directory: " + directory)
    #"stdbuf -o0 " +
    temp_c = f"bash -l '{c}'"
    temp_c = ['bash', '-lc', c]
    p = subprocess.Popen(temp_c, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, cwd=directory, bufsize=0, )
    d = ""
    current_line = ''
    buffered = ""

    while(p.poll() is None):
        #stuff=sys.stdin.read()
        # stuff !='':
        #if stuff is not None and stuff != '':
        #sys.stdout.write(f"reading stdin: {stuff}")
        if(conn.poll()):
            write_buffer(buffered)
            buffered = ''
            stuff = conn.recv()
            p.stdin.write(stuff.encode('utf-8'))

        if(not is_stuff_to_read(p)):
            continue

        character = p.stdout.read(1).decode("utf-8")
        buffered += character

        if character == "\n":
            write_buffer(buffered)
            buffered = ""

        #if not (is_buffered_line("OK", buffered) or is_buffered_line("FAILED", buffered)):
        #    buffering_line = False

    sys.stdout.flush()
    the_rest = p.stdout.read().decode("utf-8")
    the_rest = re.sub("OK", "\033[42m\033[30m{}\033[00m".format("OK".ljust(term_width)), the_rest )
    the_rest  = re.sub("FAILED(.*)\n", r"\033[41m\033[37m{}\1\033[00m\n".format("FAILED".ljust(term_width)), the_rest )

    re_text = '(?P<total_count>[\d]+) example[s]*, (?P<failure_count>[\d]+) failure[s]*'
    result = re.search(re_text, the_rest)

    #stuff=conn.recv()
    #if stuff is not None and stuff != '':
    #sys.stdout.write(f"reading stdin: {stuff}")
    #sys.stdout.flush()

    if result and int(result.group("failure_count")) > 0:
        total_match = result.group(0)
        the_rest = re.sub(total_match, "\033[41m\033[37m{}\1\033[00m".format(total_match.ljust(term_width)), the_rest)
    sys.stdout.write(the_rest)
    sys.stdout.flush()

    if bool(p.returncode):
        sys.stdout.write("\033[41m\033[37m{}\1\033[00m".format(fail_text))
    else:
        sys.stdout.write("\033[42m\033[30m{}\033[00m".format(pass_text))
    sys.stdout.flush()

    return ""

def run_commands_if_present(conn):
    things = os.listdir(COMMAND_DIRECTORY)
    if len(things) > 0:
        for thing in things:
            command_file_path = COMMAND_DIRECTORY + '/' + thing
            with open(command_file_path) as f:
                command_info = json.loads(f.read())
                os.remove(command_file_path)
                run_command(command_info['command'], command_info['directory'], conn)

def run_watcher_forever(conn):
    print("watching directory...")
    while True:
        time.sleep(.5)
        run_commands_if_present(conn)


"""
Serializers for carts models
"""
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers

from carts.models import Cart, CartItem

class CartSerializer(serializers.ModelSerializer):
    """
    Cart model serializer
    """

    class Meta:
        """
        Tells the fields to include in parsed/json object
        """

        # pylint: disable=protected-access, no-member
        model = Cart
        fields = "__all__"

class CartItemSerializer(serializers.ModelSerializer):
    """
    CartItem model serializer
    """

    def create(self, validated_data):
        """
        Creates a CartItem
        Args:
            validated_data(dict): Value containing verified data
        Returns:
            (CartItem): Value containing created CartItem instance
        """
        # pylint: disable=no-member
        cart = validated_data["cart"]
        filter_item = cart.cart_items.filter(product=validated_data["product"])
        if filter_item.exists():
            cart_item = filter_item.get()
            cart_item.quantity += validated_data["quantity"]
            if cart_item.quantity > cart_item.product.stock_quantity:
                raise serializers.ValidationError(
                    _("Product cannot be added because it's out of stock")
                )
        else:
            cart_item = CartItem.objects.create(**validated_data)
        cart_item.save()
        validated_data["cart"].update_bill()
        return cart_item

    def validate(self, attrs):
        if attrs["product"].stock_quantity < attrs["quantity"]:
            raise serializers.ValidationError(
                _("Product cannot be added because it's out of stock")
            )
        return attrs

    class Meta:
        """
        Tells model and fields to include in parsed/json object
        """

        model = CartItem
        fields = "__all__"


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Feb 28 16:39:18 2018

@author: rkoch

Here we extract all 220 bp promoter sequences for all genes. Part 2 does the same only for genes in corems

"""
import os
from Bio import SeqIO
import pickle
os.chdir('/Users/rkoch/Documents/Data_and_Scripts')

with open("MTB_genome_data_12/83332.12.fna") as f:
    record = SeqIO.parse(f, "fasta")
    genome = list(record)[0]

gene_dic = pickle.load(open('Dictionaries/gene_dic.p', 'br'))

"""
#without positions in name
gene_prom_seqs = []

for gene, positions in gene_dic.items():
    sequence = genome[positions[0]:positions[1]+1]
    sequence.id = gene
    sequence.name = ''
    sequence.description = ''
    gene_prom_seqs.append(sequence)

gene_prom_seqs.sort(key = lambda x: x.id)
SeqIO.write(gene_prom_seqs, open('MTB_genome_data_12/gene_promotors.fna', 'w'), "fasta")
"""

#with positions in name
gene_prom_seqs = []

for gene, positions in gene_dic.items():
    start = positions[0]
    end = positions[1]+1
    sequence = genome[start:end]
    sequence.id = '%s:%s-%s'%(gene, start, end)
    sequence.name = ''
    sequence.description = ''
    gene_prom_seqs.append(sequence)

gene_prom_seqs.sort(key = lambda x: x.id)
SeqIO.write(gene_prom_seqs, open('MTB_genome_data_12/gene_promotors.fna', 'w'), "fasta")

###---------------------------------------Part-2-------------------------------------------------###

#a dictionary containing for every corem all genes in the corem
corem_genes = pickle.load(open('./Dictionaries/corems_genes.p', 'rb'))
#flatten und uniquify the list of genes
vs = list(corem_genes.values())
vs = list(set([gene for l in vs for gene in l]))
corem_gene_prom_seqs = []
for gene in vs:
    positions = gene_dic[gene]
    start = positions[0]
    end = positions[1]+1
    sequence = genome[start:end]
    sequence.id = '%s:%s-%s'%(gene, start, end)
    sequence.name = ''
    sequence.description = ''
    corem_gene_prom_seqs.append(sequence)
corem_gene_prom_seqs.sort(key = lambda x: x.id)
SeqIO.write(corem_gene_prom_seqs, open('MTB_genome_data_12/corem_gene_promotors.fna', 'w'), "fasta")


#!/usr/bin/env python3
"""Directory handling assistant

Classes
-------
DHandler()
    A collection of methods that can facilitate directory handling
"""

__version__ = '1.0.0'
__author__ = 'Jaewoong Jang'

import os
import sys
import re
import argparse
from distutils.dir_util import copy_tree

class DHandler():
    """A collection of methods that can facilitate directory handling

    This class provides methods that can automate some mundane,
    repetitive directory handling tasks, such as copying a directory
    and its contents to another path.

    Attributes
    ----------
    cwd : str
        The working directory at the time of class instantiation
    border_len : int
        The number of symbols of a border line
    borders : dict
        Storage for border lines of different symbols
    funcs : dict
        Storage for the objects of directory handling methods

    Methods
    -------
    read_argv()
        Read in sys.argv using the argparse module.
    disp_func_run(msg, dfrom='', dto='', border_symb='-')
        Display a message that a function will be run.
    yn_prompt(msg='Continue? (y/n)> ')
        Invoke a y/n prompt.
    exam_exists(d, dflag='dfrom', action='exit', border_symb='*')
        Examine if the designated files exist.
    lst_subdirs(dfrom, ignore=None)
        Return the list of existing subdirectory names.
    deploy_dir(dfrom='', dto='')
        Deploy a directory, including its contents, to another path.
    deploy_empty_subdirs(dfrom='', dto='')
        Deploy empty subdirectories to another path.

    Notes
    -----
    When a directory handling method is newly defined, add the name and
    object of the method as a key-value pair to the funcs attribute;
    a user selects which directory handling method to use via the
    command line argument --func, which will then be used as a key of
    the funcs attribute.
    """

    def __init__(self):
        """Bind the objects of directory handling methods to a dict."""
        self.cwd = os.getcwd()
        self.border_len = 60
        self.borders = {s: s * self.border_len for s in ['-', '=', '+', '*']}
        self.funcs = {
            'deploy_dir': self.deploy_dir,
            'deploy_empty_subdirs': self.deploy_empty_subdirs,
            # ... Add new directory handling functions here.
        }

    def read_argv(self):
        """Read in sys.argv using the argparse module.

        Returns
        -------
        argparse.Namespace
            An object of argparse.Namespace
        """
        parser = argparse.ArgumentParser(
            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
        parser.add_argument('--func',
                            default='deploy_dir',
                            choices=self.funcs.keys(),
                            help='directory handling function')
        parser.add_argument('--dfrom',
                            required=True,
                            help=('the directory from which'
                                  + ' information will be retrieved'))
        parser.add_argument('--dto',
                            required=True,
                            help=('the directory to which the retrieved'
                                  + ' information will be applied'))
        parser.add_argument('--nopause',
                            action='store_true',
                            help=('do not pause the shell'
                                  + ' at the end of the program'))
        return parser.parse_args()

    def disp_func_run(self, msg,
                      dfrom='', dto='', border_symb='-'):
        """Display a message that a function will be run.

        Arguments
        ---------
        msg : str
            The message to be displayed
        dfrom : str
            The original directory (default '')
        dto : str
            The target directory (default '')
        border_symb : str
            A key of the borders attribute (default '-')
        """
        print(self.borders[border_symb])
        print(msg)
        print('From: [{}]'.format(dfrom))
        print('To:   [{}]'.format(dto))
        print(self.borders[border_symb])

    def yn_prompt(self,
                  msg='Continue? (y/n)> '):
        """Invoke a y/n prompt.

        Arguments
        ---------
        msg : str
            The prompt message (default 'Continue? (y/n)> ')

        Returns
        -------
        bool
            True for y or Y, False for n or Y
        """
        while True:
            yn = input(msg)
            if re.match(r'\b[yY]\b', yn):
                return True
            if re.match(r'\b[nN]\b', yn):
                return False

    def exam_exists(self, d,
                    dflag='dfrom', action='exit', border_symb='*'):
        """Examine if the designated files exist.

        Arguments
        ---------
        d : str
            The directory to be examined whether exists
        dflag : str
            A flag differentiating between dfrom and dto
        action : str
            Actions for nonexisting files (default 'exit')
        border_symb : str
            A key of the borders attribute (default '*')

        Returns
        -------
        existing_files : list
            Files found to exist out of the designated files
        """
        if not os.path.exists(d):
            print(self.borders[border_symb])
            print(f'{dflag} [{d}] not found.', end='')
            if action == 'exit':
                print(' Terminating.')
                print(self.borders[border_symb])
                sys.exit()
            print('')  # For action != 'exit'
            print(self.borders[border_symb])
            if action == 'makedirs':
                y = self.yn_prompt(msg='Create? (y/n)> ')
                if y:
                    os.makedirs(d)
                else:
                    return

    def lst_subdirs(self, dfrom,
                    ignore=None):
        """Return the list of existing subdirectory names.

        Arguments
        ---------
        dfrom : str
            The original directory
        ignore : str or list
            Subdirectories to be ignored (default None)

        Returns
        -------
        subdirs : list
            The list of subdirectories found to exist
        """
        subdirs = []
        os.chdir(dfrom)
        for f in os.listdir(dfrom):
            if os.path.isdir(f) and f not in ignore:
                subdirs.append(f)
        os.chdir(self.cwd)
        return subdirs

    def deploy_dir(self,
                   dfrom='', dto=''):
        """Deploy a directory, including its contents, to another path.

        Arguments
        ---------
        dfrom : str
            The original directory (default '')
        dto : str
            The target directory (default '')
        """
        self.disp_func_run('deploy_dir():'
                           + ' Deploying dfrom to dto...',
                           dfrom=dfrom, dto=dto)
        copy_tree(dfrom, dto)
        print('Deployment completed.')

    def deploy_empty_subdirs(self,
                             dfrom='', dto=''):
        """Deploy empty subdirectories to another path.

        Arguments
        ---------
        dfrom : str
            The original directory (default '')
        dto : str
            The target directory (default '')
        """
        self.disp_func_run('deploy_empty_subdirs():'
                           + ' Deploying empty subdirectories...',
                           dfrom=dfrom, dto=dto)
        subdirs = self.lst_subdirs(dfrom, ignore=['__pycache__', '.git'])
        # In dfrom: Exit if dfrom has no subdirectory.
        if not subdirs:
            print('dfrom has no subdirectories. Terminating.')
            return
        # Deploy the subdirectories of dfrom to dto.
        os.chdir(dto)  # CWD --> dto
        for subdir in subdirs:
            if not os.path.isdir(subdir):
                os.mkdir(subdir)
                print('[{}] created.'.format(subdir))
        os.chdir(self.cwd)  # dto --> CWD

if __name__ == '__main__':
    dh = DHandler()

    # I/O
    argv = dh.read_argv()

    # Preprocessing
    dh.exam_exists(argv.dfrom,
                   dflag='dfrom', action='exit')
    dh.exam_exists(argv.dto,
                   dflag='dto', action='makedirs')
    dh.exam_exists(argv.dto,
                   dflag='dto', action='exit')
    argv.dfrom = os.path.abspath(argv.dfrom)
    argv.dto = os.path.abspath(argv.dto)

    # Run the user-requested directory handling method.
    dh.funcs[argv.func](dfrom=argv.dfrom,
                        dto=argv.dto)
    if not argv.nopause:
        input('Press enter to exit...')


import csv
import pandas as pd
import statistics
import plotly.figure_factory as ff

df=pd.read_csv("HeightWeight.csv")
height=df["Height(Inches)"].to_list()
weight=df["Weight(Pounds)"].to_list()

#height data
mean= statistics.mean(height)
median=statistics.median(height)
stdDev=statistics.stdev(height)
mode=statistics.mode(height)

print("mean, meadian, mode, standard deviation of height is {},{},{},{}".format(mean,median,mode,stdDev))

FirstStart,FirstEnd= mean-stdDev,mean+stdDev
SecondStart,SecondEnd= mean-(2*stdDev),mean+(2*stdDev)
ThirdStart,ThirdEnd= mean-(3*stdDev),mean+(3*stdDev)
listOf1stStdDev=[result for result in height if result>FirstStart and result<FirstEnd]
listOf2ndStdDev=[result for result in height if result>SecondStart and result<SecondEnd]
listOf3rdStdDev=[result for result in height if result>ThirdStart and result<ThirdEnd]

print("{}% of data lies within 1 standard deviation".format(len(listOf1stStdDev)*100.0/len(height)))
print("{}% of data lies within 2 standard deviation".format(len(listOf2ndStdDev)*100.0/len(height)))
print("{}% of data lies within 3 standard deviation".format(len(listOf3rdStdDev)*100.0/len(height)))


#%%
import os
import sys
import json
import re
import time
import difflib
from datetime import datetime,timedelta
import requests
from dotenv import load_dotenv  # Handling Passwords and Secret Keys using Environment Variables
from pathlib import Path         #You may explicitly provide a path to your .env file.

env_path=Path('.') / '.env'
load_dotenv(dotenv_path=env_path)

from logging import log
from logging import Logger

#%%

# ====================================================
# defining global variables
# ====================================================

webhook_url=os.environ['WEBHOOK_URL']
slack_bot_user_oauth_token= os.environ['SLACK_BOT_USER_OAUTH_TOKEN']
slack_username= os.environ['SLACK_USERNAME']
slack_channelname= os.environ['SLACK_CHANNELNAME']
github_token= os.environ['GITHUBTOKEN']
schema_file_url= os.environ['SCHEMA_FILE_URL']

now=datetime.now()
current_date_and_time=now.strftime("%D %H:%M:%S")
current_date= now.strftime("%y_%m_%d")

yesterday= now-timedelta(days=1)
yesterday=yesterday.strftime("%y_%m_%d")

# define file names to be compared
file_name_1=f'schema_{yesterday}.rb'
file_name_2=f'schema_{current_date}.rb'

# create log files
change_log_filename = f'change_log_{current_date}.txt'
project_name_and_date= ' Database Schema Change: ' + current_date_and_time + '\n\n'
change_log= open(change_log_filename,'w')
change_log.write(project_name_and_date)

#%%

# ====================================================
# defining functions
# ====================================================

#log process

def _get_logger(name) -> Logger:
    from logging import Formatter,Logger,basicConfig,getLogger,StreamHandler,DEBUG,INFO

    date_format = '%m,%d,%Y %I:%M:%S %p'
    log_format= '%(asctime)s %(levelname)-8s %(name)-15s %(message)s'
    basicConfig(filename= 'etl.log',format=log_format,datefmt=date_format)

    log: Logger = getLogger('Logging')
    log.setLevel(INFO)
    ch=StreamHandler()
    ch.setLevel(DEBUG)
    formatter= Formatter(log_format)
    ch.setFormatter(formatter)
    log.addHandler(ch)

    return log

log: Logger = _get_logger('Logging:')

# =========

def github_download(token,url):

    """
    Call     : github_download(token,url)
    Returns   : file_content
    """

    file_url= url + token

    try:
        # log.info("started download")
        response= requests.get(file_url)

        if response==200:
            file_content=response.text
            #log.info("Downloaded file")
            return file_content

    except:

        status=requests.get(file_url).status_code
        print('Error: ',status)
        return False

#%%
# =========

def write_file(data,file_name):

    """
    Call     :  write_file(data,file_name)
    Returns   : True if able to write the data as a file,otherwise False
    """

    try:
        file= open(file_name,'w')
        file.write(data)
        return True

    except:
        print(f'Error: Could not write {file_name}')
        return False

#%%
# =========

#parse table names from the schema file

def parse_tables(schema_rb):
    """
    Call     :  parse_tables(schema_rb)
    Returns   : A dictionary of table names as key and column definitions from the DDL
    """

    tables={}
    for i in schema_rb:

        try:
            if "create_table" in i:
                table_name= re.search('".*"',i).group().replace('"','') #parse table name

                #each table name will be a key in the table_names dictionary
                tables.update({table_name:[]})

            #following rows will be its value
            elif "end" not in i:
                tables[table_name].append(i.replace('\n',''))

        except:
            continue

    return tables

#%%
#======================

def get_file_content(file_name):
    content= ''

    with open(file_name,'r') as f:
        content=f.read()
    return content

def post_file_to_slack( text, file_name, channelname, file_bytes, file_type= None, title=None):
    return requests.post(
        'https://slack.com/api/files.upload',
        {
            'token': slack_bot_user_oauth_token,
            'filename': file_name,
            'channels': channelname,
            'filetype': 'text',
            'initial_comment': None,
            'title': None

        },
    files= {'file':file_bytes}).json()

# %%
# ==================================
# 1. Download new schema file
# ==================================

try:
    data= github_download(github_token, schema_file_url)
     # write_file(data, file_name_2)

except:

    log.info('Error: Could not download schema.rb')

# %%
# ==================================
# 2. Read files
# ==================================

with open(file_name_1,'r') as file_1:
  file_1_text=file_1.readlines()

with open(file_name_2,'r') as file_2:
  file_2_text=file_2.readlines()

# %%
# ==================================
# 3.  Test 1:  Compare if there are any changes
# ==================================

try:
    assert(file_1_text==file_2_text)
    change_log.write('Test 0: PASS  -  No Schema Change Detected\n')
    log.info('Test 0: PASS  -  No Schema Change Detected\n')

except:
    change_log.write('Test 0: FAIL  - Schema Change Detected\n')
    log.info('Test 0: FAIL  - Schema Change Detected\n')

# ==================================
# 3.  Test 2:  Compare if tables changed
# ==================================

#table_names in table_1 or old file
previous_schema= parse_tables(file_1_text)
new_schema= parse_tables(file_2_text)

previous_schema_tables= set(previous_schema.keys())

#table_names in table_1 or old file
new_schema_tables= set(new_schema.keys())

    # compare files now;

try:
    assert(previous_schema_tables==new_schema_tables)
    change_log.write('Test 1: PASS  -  No changes made to the table names\n')
    log.info('Test 1: PASS  -  No changes made to the table names\n')

except:
    change_log.write('Test 1: FAIL  - Table Name Change Detected\n')
    log.info('Test 0: FAIL  - Table Name Change Detected\n')

    if len(previous_schema_tables.difference(new_schema_tables))>0:
        change_log.write(
            f"""
        Deleted Tables:
                {previous_schema_tables.difference(new_schema_tables)}\n
            """
        )

    if len(new_schema_tables.difference(previous_schema_tables))>0:
        change_log.write(
            f"""
        New Tables:
                {new_schema_tables.difference(previous_schema_tables)}\n
            """
        )

# ==================================
# 3.  Test 3:  Compare column definitions
# ==================================

table_counter=0
for table_name in previous_schema_tables:
    try:
        if table_name not in new_schema_tables:
            continue
        else:
            rows_1=set(previous_schema[table_name])
            rows_2=set(new_schema[table_name])
            assert(rows_1==rows_2)
            #change_log.write('Test 2: PASS  - No column changes were detected ')
            continue

    except:
        if table_name not in new_schema_tables:
            continue
        else:
            rows_1=set(previous_schema[table_name])
            rows_2=set(new_schema[table_name])

            #isolate the error
            if table_counter==0:
                #change_log.write('\n Test 2: FAIL - Column changes detected \n')
                table_counter+=1
            change_log.write(f"""
        Table Name: {table_name} """ )

            if len(rows_1.difference(rows_2))>0:
                #Possible Deletions
                change_log.write(f"""
        Column Differences - Previous Schema:
                {rows_1.difference(rows_2)}
                """
                )

            if len(rows_2.difference(rows_1))>0:
                #Possible Additiona
                change_log.write(f"""
        Column Differences - New Schema:
                {str(rows_2.difference(rows_1))}
                """
                )
change_log.close()

#%%
# ==================================
# 4.  Post Log to Slack
# ==================================

post_file_to_slack(
    text=None,
    file_name=change_log_filename,
    channelname=slack_channelname,
    file_bytes=get_file_content(change_log_filename)
)