from typing import Any
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import status, filters, viewsets
from rest_framework.decorators import action
from rest_framework.permissions import AllowAny, IsAuthenticated
from rest_framework.response import Response
from django.db import transaction
#from rest_framework.settings import api_settings

from django.db.models import Sum, Count
import json
from django.core.files import File

#from api.permission import IsStaff
from api.permission import IsAdmin
from api.models import Categoria
from api.serializers import CategoriaSerializer, CategoriaRegistroSerializer

class CategoriaViewset(viewsets.ModelViewSet):
    queryset = Categoria.objects.filter(activo=True)
    #permission_classes = (IsStaff,)

    filter_backends = (DjangoFilterBackend, filters.SearchFilter, filters.OrderingFilter)
    filter_fields = ("nombre",)
    search_fields = ("nombre",)
    ordering_fields = ("nombre",)

    def get_serializer_class(self):
        """Define serializer for API"""
        if self.action == 'list' or self.action == 'retrieve':
            return CategoriaSerializer
        else:
            return CategoriaRegistroSerializer

    def get_permissions(self):
        """" Define permisos para este recurso """
        if self.action == "create" or self.action == "update" or self.action == "destroy":
            permission_classes = [IsAdmin]
        else:
            permission_classes = [IsAuthenticated]
        return [permission() for permission in permission_classes]

    def list(self, request, *args, **kwargs):
        queryset = self.filter_queryset(Categoria.objects.filter(activo=True))
        #queryset = DetalleProducto.objects.filter(activo=True)
        serializer = CategoriaSerializer(queryset, many=True)

        page = request.GET.get('page')

        if page is not None:
            serializer = self.get_serializer(page, many=True)
            data = serializer.data
            return self.get_paginated_response(data)

        #print(serializer.data)

        return Response(serializer.data, status=status.HTTP_200_OK)

    def create(self, request):
        try:
            with transaction.atomic():
                #user = request.data
                #imagen = data.get('imagen')
                data = request.data
                print(data)

                #serializer = CatedraticoRegistroSerializer(data=request.data)
                verify = CategoriaRegistroSerializer(data=data)
                if verify.is_valid():

                    """if imagen is not None:
                        imagen = File(imagen)

                        categoria = Categoria.objects.create(
                            nombre=data.get('nombre'),
                            imagen=imagen
                        )
                    elif imagen is None:
                        categoria = Categoria.objects.create(
                            nombre=data.get('nombre'),
                        )"""
                    categoria = Categoria.objects.create(
                            nombre=data.get('nombre'),
                            imagen=data.get('imagen'),
                    )

                else:
                    #print("Error en la verificación")
                    return Response({"detail":str(verify.errors)}, status=status.HTTP_400_BAD_REQUEST)
            return Response(verify.data, status=status.HTTP_200_OK)
        except Exception as e:
            return Response({'detail':str(e)}, status=status.HTTP_400_BAD_REQUEST)

    def update(self, request, pk):
        try:
            with transaction.atomic():
                data = request.data

                print(data)

                #imagen = data.get("imagen")
                #data = json.loads(data["data"])

                #print('imagen', imagen)

                verify = CategoriaRegistroSerializer(data=data)
                #if verify.is_valid():

                categoria = Categoria.objects.get(pk=pk)

                """if categoria.imagen is None:
                    if imagen is not None:
                        categoria.imagen = File(imagen)
                elif categoria.imagen is not None:
                    if imagen is not None:
                        categoria.imagen.delete()
                        categoria.imagen = File(imagen)"""

                categoria.nombre = data.get('nombre')
                categoria.imagen = data.get('imagen')
                categoria.save()
                #else:
                #    return Response({"detail_serializer":str(verify.errors)}, status=status.HTTP_400_BAD_REQUEST)
            return Response( status=status.HTTP_200_OK)
        except Exception as e:
            return Response({'detail':str(e)}, status=status.HTTP_400_BAD_REQUEST)

    """@action(methods=["get"], detail=False)
    def total_cursos(self, request, *args, **kwargs):
        user = request.user
        queryset = Categoria.objects.filter(
            activo=True
            ).aggregate(
                total=Count(
                    'id'
                    ),
                )
        return Response(queryset, status=status.HTTP_200_OK)"""


from fastapi import FastAPI
from dotenv import dotenv_values
# from pymongo import MongoClient
from app.routes import router
from pymongo import mongo_client

config = dotenv_values(".env")

app = FastAPI()

# @app.get("/")
# async def root():
#     return {"message": "API using Fast API and pymongo"}

@app.on_event("startup")
def startup_db_client():

    app.mongodb_client = mongo_client.MongoClient(config["DB_URL"])
    app.database = app.mongodb_client[config["DB_NAME"]]
    print("Connected to the MongoDB database!")

@app.on_event("shutdown")
def shutdown_db_client():
    app.mongodb_client.close()

app.include_router(router, tags=["list"], prefix="")


from PIL import Image
import os
import numpy as np
import sys
import Augmentor

"""
This file converts a dataset of images into different size and labels
used for preprocessing a Conditional WGAN Dataset
"""

# Defined variables with paths
out_path = "out/"
in_path = "train/"
labels = ["_small", "_big"]
img_format = ".png"  # might need to be changed depending on the dataset

# This method converts every image from a folder
# into a 64x64 Binary Image and assigns a label
# given the percentage of road in it.
# @param in_path The path to the images
# @param out_path The path to the output folder
# @param labaels The labels for the images
# @param pct The percentage that decides the labels
def convert_data(in_path, out_path, labels, pct=35):
    counter = 1
    print("Converting...")
    for im_path in os.listdir(in_path):
        if img_format not in im_path: continue
        # os.rename(in_path+im_path, str(counter)+".jpg")  # if renaming is needed
        img = Image.open(in_path+im_path)
        np.set_printoptions(threshold=sys.maxsize)
        img = np.array(img.resize((64, 64), Image.BILINEAR))
        count = np.count_nonzero(img)
        pct_count = (count/float(img.shape[0]*img.shape[1])) * 100.0  # Get percentage of road
        img = Image.fromarray(np.uint8(np.where(np.array(img) > 0, 1, 0)*255))
        if pct_count > pct: img.save(out_path + str(counter) + labels[1] + ".jpg")
        else: img.save(out_path + str(counter) + labels[0] + ".jpg")
        counter += 1
    print("Finished!")

# This method augments the data
# by rotating and zooming
# @param path The path to the images
def augment_data(path):
    print("Augmenting...")
    augmentor = Augmentor.Pipeline(path)
    augmentor.rotate90(1)
    augmentor.process()
    augmentor.rotate270(1)
    augmentor.process()
    print("Finished!")

# 1st) Augment_data
# augment_data(in_path[:-1])  # Remove last \
# 2nd) Convert data
# Do only after moving output folder to main folder after augmentation
convert_data(in_path, out_path, labels)


import pandas as pd
import matplotlib.pyplot as plt
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_validate
from sklearn.preprocessing import MinMaxScaler
import numpy as np

#Reads the data
df = pd.read_csv("Task3 - dataset - HIV RVG.csv")

#Plots a box plot by the participant condition
dfBoxPlot = df[["Alpha", "Participant Condition"]]
dfBoxPlot.boxplot(by = "Participant Condition")

#Plots a density plot by the participant condition
dfDensityPlot = df[["Beta","Participant Condition"]].pivot(columns = "Participant Condition", values = "Beta")
dfDensityPlot.plot.density(by = "Participant Condition")

#Outputs summary statistics
print(df.describe())

#Scaler to normalize the data between the values -1 and 1
scaler = MinMaxScaler(feature_range = (-1,1))

#Fits the scaler to the numerical dataset columns
scaler = scaler.fit(df.iloc[:,0:8])
scaler = scaler.transform(df.iloc[:,0:8])

#List of different iteration values to alter the max iterations of the neural network and random forest model
neuralIterations = [100,200,300,400,500,600,700,800,900,1000]
forestIterations = [5,10]

#Lists to append the accuracy of each model
accuracyOfNeuralNetwork = []
accuracyOfRandomForest = []

xTrain, xTest, yTrain, yTest = train_test_split(df.iloc[:,0:8],df.iloc[:,8],test_size=0.1,shuffle = True) #Creates train teest split

#Loops through the neural network iterations
for i in neuralIterations:
    #Creates a neural network model with varying max iterations
    clf = MLPClassifier(solver='sgd', alpha=1e-5, hidden_layer_sizes=(500, 500), activation = "logistic",max_iter=i)
    #Fits the model to the training and testing set
    clf.fit(xTrain,yTrain)
    #Evaluates the accuracy of the model
    evaluate = clf.score(xTest,yTest)
    print("Neural network iteration", i ," Accuracy: ",evaluate) #Outputs accuracy and number of iterations
    #Appends the accuracy to a list
    accuracyOfNeuralNetwork.append(evaluate)
#Loops through the random forest iterations
for i in forestIterations:
    #Creates a random forest model with varying
    clf = RandomForestClassifier(max_depth=2, n_estimators = 1000,min_samples_leaf = i)
    #Fits the model to the training and testing set
    clf.fit(xTrain,yTrain)
    #Evaluates the accuracy of the model
    evaluate = clf.score(xTest,yTest)
    print("Random forest iteration", i ," Accuracy: ",evaluate)#Outputs accuracy and number of iterations
    #Appends the accuracy to a list
    accuracyOfRandomForest.append(evaluate)

#Plots the accuracy of the neural network against the number of iterations
plt.figure()
plt.plot(neuralIterations,accuracyOfNeuralNetwork)
plt.xlabel("Epochs")
plt.ylabel("Accuracy")

#Plots the accuracy of the neural network against the number of iterations
plt.figure()
plt.plot(forestIterations,accuracyOfRandomForest)
plt.xlabel("Epochs")
plt.ylabel("Accuracy")

#Cross validation
def crossValidate(clf,df,model,layers):

    #Cross validates 10 times
    scores = cross_validate(clf, df.iloc[:,0:8], df.iloc[:,8],cv = 10)['test_score']
    print(model + layers +" Score: ",scores.mean()) #Outputs the mean of cross validation

#Creates three neural networks with hidden layers 50,500,1000
neuralClf1 = MLPClassifier(solver='sgd', alpha=1e-5, hidden_layer_sizes=(50,50), activation = "logistic",max_iter=1000)
neuralClf2 = MLPClassifier(solver='sgd', alpha=1e-5, hidden_layer_sizes=(500,500), activation = "logistic",max_iter=1000)
neuralClf3 = MLPClassifier(solver='sgd', alpha=1e-5, hidden_layer_sizes=(1000,1000), activation = "logistic",max_iter=1000)

#Creates three random forest classifiers with hidden layers 50,500,1000
ranForestClf1 = RandomForestClassifier(max_depth=2, random_state=0, n_estimators = 50,min_samples_leaf = 10)
ranForestClf2 = RandomForestClassifier(max_depth=2, random_state=0, n_estimators = 500,min_samples_leaf = 10)
ranForestClf3 = RandomForestClassifier(max_depth=2, random_state=0, n_estimators = 10000,min_samples_leaf = 10)

#Cross validates with varying models
crossValidate(neuralClf1,df,"Neural Network"," layers: 50")
crossValidate(neuralClf2,df,"Neural Network"," layers: 500")
crossValidate(neuralClf3,df,"Neural Network"," layers: 1000")

crossValidate(ranForestClf1,df,"Random Forest"," trees: 50")
crossValidate(ranForestClf2,df,"Random Forest"," trees: 500")
crossValidate(ranForestClf3,df,"Random Forest"," trees: 10000")


import gzip
from collections import Counter, defaultdict, namedtuple
import sys
import os

ped_file = sys.argv[1] #'../data/160826.ped'
id_mapping = sys.argv[2] #'../data/160826.iHART.db.query.csv'
variant_dir = sys.argv[3] #'../other_sv_calls/genomeStrip'
out_file = sys.argv[4] # ../other_sv_calls/

# Affection (0=unknown; 1=unaffected; 2=affected)
child_id_to_affected = dict()
child_id_to_sex = dict()
child_id_to_parent_ids = dict()

Deletion = namedtuple('Deletion', ['chrom', 'start', 'end'])

with open(ped_file, 'r') as f:
    for line in f:
        pieces = line.strip().split('\t')
        if len(pieces) >= 6:
            fam_id, child_id, f_id, m_id, sex, disease_status = pieces[0:6]
            child_id_to_affected[child_id] = disease_status
            child_id_to_sex[child_id] = sex
            child_id_to_parent_ids[child_id] = (f_id, m_id)

id_switch = dict()
with open(id_mapping, 'r') as f:
    for line in f:
        pieces = line.strip().split(',')
        id_switch[pieces[1]] = pieces[2]

sample_id_to_filename = dict()
for filename in os.listdir(variant_dir):
    if filename.endswith(".bed"):
        sample_id = filename.split('.')[0]
        if sample_id in id_switch:
            sample_id_to_filename[id_switch[sample_id]] = os.path.join(variant_dir, filename)
print('Sample IDs', len(sample_id_to_filename))

# load parental deletions
parent_deletions = dict()
for parent_id in sum([list(x) for x in child_id_to_parent_ids.values()], []):
    if parent_id in sample_id_to_filename:
        deletions = []
        with open(sample_id_to_filename[parent_id], 'r') as f:
            for line in f:
                pieces = line.strip().split('\t')
                if pieces[3] == 'DEL':
                    deletions.append(Deletion(pieces[0], int(pieces[1]), int(pieces[2])))
        parent_deletions[parent_id] = deletions
print('Parental deletions loaded')

with open(out_file, 'w+') as outf:
    for child_id, (f_id, m_id) in child_id_to_parent_ids.items():
        if child_id in sample_id_to_filename and f_id in parent_deletions:
            par_deletions = parent_deletions[f_id]
            current_chrom = '1'
            current_chrom_dels = [x for x in par_deletions if x[0] == current_chrom]
            with open(sample_id_to_filename[child_id], 'r') as f:
                for line in f:
                    pieces = line.strip().split('\t')
                    if pieces[3] == 'DEL':
                        child_del = Deletion(pieces[0], int(pieces[1]), int(pieces[2]))
                        if child_del.chrom != current_chrom:
                            current_chrom = pieces[0]
                            current_chrom_dels = [x for x in par_deletions if x.chrom == current_chrom]

                        for d in current_chrom_dels:
                            if d.end < child_del.start:
                                overlap = 0
                            elif child_del.end < d.start:
                                overlap = 0
                                break
                            elif d.start <= child_del.start and d.end >= child_del.end:
                                overlap = child_del.end-child_del.start+1
                            elif child_del.start <= d.start and child_del.end >= d.end:
                                overlap = d.end-d.start+1
                            elif child_del.start < d.end:
                                overlap = d.end - child_del.start + 1
                            elif d.start < child_del.end:
                                overlap = child_del.end - d.start + 1
                            if overlap/(child_del.end-child_del.start+1) >= 0.5 and overlap/(d.end-d.start+1) >= 0.5:
                                outf.write('\t'.join([d.chrom, str(d.start), str(d.end), str(child_del.start), str(child_del.end), '', '', '', '', '%s-%s' % (m_id, child_id)]) + '\n')

        if child_id in sample_id_to_filename and m_id in parent_deletions:
            par_deletions = parent_deletions[m_id]
            current_chrom = '1'
            current_chrom_dels = [x for x in par_deletions if x[0] == current_chrom]
            with open(sample_id_to_filename[child_id], 'r') as f:
                for line in f:
                    pieces = line.strip().split('\t')
                    if pieces[3] == 'DEL':
                        child_del = Deletion(pieces[0], int(pieces[1]), int(pieces[2]))
                        if child_del.chrom != current_chrom:
                            current_chrom = pieces[0]
                            current_chrom_dels = [x for x in par_deletions if x.chrom == current_chrom]

                        for d in current_chrom_dels:
                            if d.end < child_del.start:
                                overlap = 0
                            elif child_del.end < d.start:
                                overlap = 0
                                break
                            elif d.start <= child_del.start and d.end >= child_del.end:
                                overlap = child_del.end-child_del.start+1
                            elif child_del.start <= d.start and child_del.end >= d.end:
                                overlap = d.end-d.start+1
                            elif child_del.start < d.end:
                                overlap = d.end - child_del.start + 1
                            elif d.start < child_del.end:
                                overlap = child_del.end - d.start + 1
                            if overlap/(child_del.end-child_del.start+1) >= 0.5 and overlap/(d.end-d.start+1) >= 0.5:
                                outf.write('\t'.join([d.chrom, str(d.start), str(d.end), str(child_del.start), str(child_del.end), '', '', '', '', '%s-%s' % (m_id, child_id)]) + '\n')


#!/usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import asyncio
import configparser
import json
import logging
import os
import shutil
import tempfile
import time
from contextlib import AsyncExitStack
from distutils.util import strtobool
from pathlib import Path

import aiohttp
from mardor.reader import MarReader
from mardor.signing import get_keysize
from scriptworker.utils import get_hash, retry_async

log = logging.getLogger(__name__)

ROOT_URL = os.environ.get(
    "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com"
)
QUEUE_PREFIX = f"{ROOT_URL}/api/queue/"
ALLOWED_URL_PREFIXES = (
    "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/",
    "http://download.cdn.mozilla.net/pub/firefox/nightly/",
    "http://ftp.mozilla.org/",
    "http://download.mozilla.org/",
    "https://archive.mozilla.org/",
    "http://archive.mozilla.org/",
    QUEUE_PREFIX,
)
STAGING_URL_PREFIXES = (
    "http://ftp.stage.mozaws.net/",
    "https://ftp.stage.mozaws.net/",
)

BCJ_OPTIONS = {
    "x86": ["--x86"],
    "x86_64": ["--x86"],
    "aarch64": [],
    # macOS Universal Builds
    "macos-x86_64-aarch64": [],
}

def verify_signature(mar, cert):
    log.info("Checking %s signature", mar)
    with open(mar, "rb") as mar_fh:
        m = MarReader(mar_fh)
        if not m.verify(verify_key=cert):
            raise ValueError(
                "MAR Signature invalid: %s (%s) against %s", mar, m.signature_type, cert
            )

def process_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument("--artifacts-dir", required=True)
    parser.add_argument("--signing-cert", type=argparse.FileType("rb"), required=True)
    parser.add_argument("--task-definition", required=True, type=argparse.FileType("r"))
    parser.add_argument(
        "--allow-staging-prefixes",
        action="store_true",
        default=strtobool(os.environ.get("FUNSIZE_ALLOW_STAGING_PREFIXES", "false")),
        help="Allow files from staging buckets.",
    )
    parser.add_argument(
        "-q",
        "--quiet",
        dest="log_level",
        action="store_const",
        const=logging.INFO,
        default=logging.DEBUG,
    )
    parser.add_argument(
        "--arch",
        type=str,
        required=True,
        choices=BCJ_OPTIONS.keys(),
        help="The archtecture you are building.",
    )
    return parser.parse_args()

def validate_mar_channel_id(mar, channel_ids):
    log.info("Checking %s for MAR_CHANNEL_ID %s", mar, channel_ids)
    # We may get a string with a list representation, or a single entry string.
    channel_ids = set(channel_ids.split(","))

    product_info = MarReader(open(mar, "rb")).productinfo
    if not isinstance(product_info, tuple):
        raise ValueError(
            "Malformed product information in mar: {}".format(product_info)
        )

    found_channel_ids = set(product_info[1].split(","))

    if not found_channel_ids.issubset(channel_ids):
        raise ValueError(
            "MAR_CHANNEL_ID mismatch, {} not in {}".format(product_info[1], channel_ids)
        )

    log.info("%s channel %s in %s", mar, product_info[1], channel_ids)

async def retry_download(*args, semaphore=None, **kwargs):  # noqa: E999
    """Retry download() calls."""
    async with AsyncExitStack() as stack:
        if semaphore:
            await stack.enter_async_context(semaphore)
        await retry_async(
            download,
            retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError),
            args=args,
            kwargs=kwargs,
        )

def verify_allowed_url(mar, allowed_url_prefixes):
    if not any(mar.startswith(prefix) for prefix in allowed_url_prefixes):
        raise ValueError(
            "{mar} is not in allowed URL prefixes: {p}".format(
                mar=mar, p=allowed_url_prefixes
            )
        )

async def download(url, dest, mode=None):  # noqa: E999
    log.info("Downloading %s to %s", url, dest)
    chunk_size = 4096
    bytes_downloaded = 0
    async with aiohttp.ClientSession(raise_for_status=True) as session:
        start = time.time()
        async with session.get(url, timeout=120) as resp:
            # Additional early logging for download timeouts.
            log.debug("Fetching from url %s", resp.url)
            for history in resp.history:
                log.debug("Redirection history: %s", history.url)
            log.debug("Headers for %s: %s", resp.url, resp.headers)
            if "Content-Length" in resp.headers:
                log.debug(
                    "Content-Length expected for %s: %s",
                    url,
                    resp.headers["Content-Length"],
                )
            log_interval = chunk_size * 1024
            with open(dest, "wb") as fd:
                while True:
                    chunk = await resp.content.read(chunk_size)
                    if not chunk:
                        break
                    fd.write(chunk)
                    bytes_downloaded += len(chunk)
                    log_interval -= len(chunk)
                    if log_interval <= 0:
                        log.debug("Bytes downloaded for %s: %d", url, bytes_downloaded)
                        log_interval = chunk_size * 1024
            end = time.time()
            log.info(
                "Downloaded %s, %s bytes in %s seconds: sha256:%s",
                url,
                bytes_downloaded,
                int(end - start),
                get_hash(dest, hash_alg="sha256"),
            )
            if mode:
                log.info("chmod %o %s", mode, dest)
                os.chmod(dest, mode)

async def download_buildsystem_bits(partials_config, downloads, tools_dir):
    """Download external tools needed to make partials."""

    # We're making the assumption that the "to" mar is the same for all,
    # as that's the way this task is currently used.
    to_url = extract_download_urls(partials_config, mar_type="to").pop()

    repo = get_option(
        downloads[to_url]["extracted_path"],
        filename="platform.ini",
        section="Build",
        option="SourceRepository",
    )
    revision = get_option(
        downloads[to_url]["extracted_path"],
        filename="platform.ini",
        section="Build",
        option="SourceStamp",
    )

    urls = {
        "make_incremental_update.sh": f"{repo}/raw-file/{revision}/tools/"
        "update-packaging/make_incremental_update.sh",
        "common.sh": f"{repo}/raw-file/{revision}/tools/update-packaging/common.sh",
        "mar": "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/"
        "latest-mozilla-central/mar-tools/linux64/mar",
        "mbsdiff": "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/"
        "latest-mozilla-central/mar-tools/linux64/mbsdiff",
    }
    for filename, url in urls.items():
        filename = tools_dir / filename
        await retry_download(url, dest=filename, mode=0o755)

def find_file(directory, filename):
    log.debug("Searching for %s in %s", filename, directory)
    return next(Path(directory).rglob(filename))

def get_option(directory, filename, section, option):
    log.info("Extracting [%s]: %s from %s/**/%s", section, option, directory, filename)
    f = find_file(directory, filename)
    config = configparser.ConfigParser()
    config.read(f)
    rv = config.get(section, option)
    log.info("Found %s", rv)
    return rv

def extract_download_urls(partials_config, mar_type):
    """Extract a set of urls to download from the task configuration.

    mar_type should be one of "from", "to"
    """
    return {definition[f"{mar_type}_mar"] for definition in partials_config}

async def download_and_verify_mars(partials_config, allowed_url_prefixes, signing_cert):
    """Download, check signature, channel ID and unpack MAR files."""
    # Separate these categories so we can opt to perform checks on only 'to' downloads.
    from_urls = extract_download_urls(partials_config, mar_type="from")
    to_urls = extract_download_urls(partials_config, mar_type="to")
    tasks = list()
    downloads = dict()

    semaphore = asyncio.Semaphore(2)  # Magic 2 to reduce network timeout errors.
    for url in from_urls.union(to_urls):
        verify_allowed_url(url, allowed_url_prefixes)
        downloads[url] = {
            "download_path": Path(tempfile.mkdtemp()) / Path(url).name,
        }
        tasks.append(
            retry_download(url, downloads[url]["download_path"], semaphore=semaphore)
        )

    await asyncio.gather(*tasks)

    for url in downloads:
        # Verify signature, but not from an artifact as we don't
        # depend on the signing task
        if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION") and not url.startswith(
            QUEUE_PREFIX
        ):
            verify_signature(downloads[url]["download_path"], signing_cert)

        # Only validate the target channel ID, as we update from beta->release
        if url in to_urls:
            validate_mar_channel_id(
                downloads[url]["download_path"], os.environ["MAR_CHANNEL_ID"]
            )

        downloads[url]["extracted_path"] = tempfile.mkdtemp()
        with open(downloads[url]["download_path"], "rb") as mar_fh:
            log.info(
                "Unpacking %s into %s",
                downloads[url]["download_path"],
                downloads[url]["extracted_path"],
            )
            m = MarReader(mar_fh)
            m.extract(downloads[url]["extracted_path"])

    return downloads

async def run_command(cmd, cwd="/", env=None, label=None, silent=False):
    log.info("Running: %s", cmd)
    if not env:
        env = dict()
    process = await asyncio.create_subprocess_shell(
        cmd,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
        cwd=cwd,
        env=env,
    )
    if label:
        label = "{}: ".format(label)
    else:
        label = ""

    async def read_output(stream, label, printcmd):
        while True:
            line = await stream.readline()
            if line == b"":
                break
            printcmd("%s%s", label, line.decode("utf-8").rstrip())

    if silent:
        await process.wait()
    else:
        await asyncio.gather(
            read_output(process.stdout, label, log.info),
            read_output(process.stderr, label, log.warning),
        )
        await process.wait()

async def generate_partial(from_dir, to_dir, dest_mar, mar_data, tools_dir, arch):
    log.info("Generating partial %s", dest_mar)
    env = os.environ.copy()
    env["LC_ALL"] = "C"
    env["MAR"] = tools_dir / "mar"
    env["MBSDIFF"] = tools_dir / "mbsdiff"
    if arch:
        env["BCJ_OPTIONS"] = " ".join(BCJ_OPTIONS[arch])
    env["MOZ_PRODUCT_VERSION"] = mar_data["version"]
    env["MAR_CHANNEL_ID"] = mar_data["MAR_CHANNEL_ID"]
    env["BRANCH"] = mar_data["branch"]

    make_incremental_update = tools_dir / "make_incremental_update.sh"
    cmd = f"{make_incremental_update} {dest_mar} {from_dir} {to_dir}"

    await run_command(cmd, cwd=dest_mar.parent, env=env, label=dest_mar.name)
    validate_mar_channel_id(dest_mar, mar_data["MAR_CHANNEL_ID"])

async def manage_partial(
    partial_def, artifacts_dir, tools_dir, downloads, semaphore, arch=None
):
    from_url = partial_def["from_mar"]
    to_url = partial_def["to_mar"]
    from_path = downloads[from_url]["extracted_path"]
    to_path = downloads[to_url]["extracted_path"]

    mar_data = {
        "MAR_CHANNEL_ID": os.environ["MAR_CHANNEL_ID"],
        "version": get_option(
            to_path, filename="application.ini", section="App", option="Version"
        ),
        "appName": get_option(
            from_path, filename="application.ini", section="App", option="Name"
        ),
        # Use Gecko repo and rev from platform.ini, not application.ini
        "repo": get_option(
            to_path, filename="platform.ini", section="Build", option="SourceRepository"
        ),
        "revision": get_option(
            to_path, filename="platform.ini", section="Build", option="SourceStamp"
        ),
        "locale": partial_def["locale"],
        "from_mar": partial_def["from_mar"],
        "from_size": os.path.getsize(downloads[from_url]["download_path"]),
        "from_hash": get_hash(downloads[from_url]["download_path"], hash_alg="sha512"),
        "from_buildid": get_option(
            from_path, filename="application.ini", section="App", option="BuildID"
        ),
        "to_mar": partial_def["to_mar"],
        "to_size": os.path.getsize(downloads[to_url]["download_path"]),
        "to_hash": get_hash(downloads[to_url]["download_path"], hash_alg="sha512"),
        "to_buildid": get_option(
            to_path, filename="application.ini", section="App", option="BuildID"
        ),
        "mar": partial_def["dest_mar"],
    }
    # if branch not set explicitly use repo-name
    mar_data["branch"] = partial_def.get("branch", Path(mar_data["repo"]).name)

    for field in (
        "update_number",
        "previousVersion",
        "previousBuildNumber",
        "toVersion",
        "toBuildNumber",
    ):
        if field in partial_def:
            mar_data[field] = partial_def[field]

    dest_mar = Path(artifacts_dir) / mar_data["mar"]

    async with semaphore:
        await generate_partial(from_path, to_path, dest_mar, mar_data, tools_dir, arch)

    mar_data["size"] = os.path.getsize(dest_mar)
    mar_data["hash"] = get_hash(dest_mar, hash_alg="sha512")
    return mar_data

async def async_main(args, signing_cert):
    tasks = []

    allowed_url_prefixes = list(ALLOWED_URL_PREFIXES)
    if args.allow_staging_prefixes:
        allowed_url_prefixes += STAGING_URL_PREFIXES

    task = json.load(args.task_definition)

    downloads = await download_and_verify_mars(
        task["extra"]["funsize"]["partials"], allowed_url_prefixes, signing_cert
    )

    tools_dir = Path(tempfile.mkdtemp())
    await download_buildsystem_bits(
        partials_config=task["extra"]["funsize"]["partials"],
        downloads=downloads,
        tools_dir=tools_dir,
    )

    # May want to consider os.cpu_count() if we ever run on osx/win.
    # sched_getaffinity is the list of cores we can run on, not the total.
    semaphore = asyncio.Semaphore(len(os.sched_getaffinity(0)))
    for definition in task["extra"]["funsize"]["partials"]:
        tasks.append(
            asyncio.ensure_future(
                retry_async(
                    manage_partial,
                    retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError),
                    kwargs=dict(
                        partial_def=definition,
                        artifacts_dir=args.artifacts_dir,
                        tools_dir=tools_dir,
                        arch=args.arch,
                        downloads=downloads,
                        semaphore=semaphore,
                    ),
                )
            )
        )
    manifest = await asyncio.gather(*tasks)

    for url in downloads:
        downloads[url]["download_path"].unlink()
        shutil.rmtree(downloads[url]["extracted_path"])
    shutil.rmtree(tools_dir)

    return manifest

def main():
    args = process_arguments()

    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
    log.setLevel(args.log_level)

    signing_cert = args.signing_cert.read()
    assert get_keysize(signing_cert) == 4096

    artifacts_dir = Path(args.artifacts_dir)
    if not artifacts_dir.exists():
        artifacts_dir.mkdir()

    loop = asyncio.get_event_loop()
    manifest = loop.run_until_complete(async_main(args, signing_cert))
    loop.close()

    manifest_file = artifacts_dir / "manifest.json"
    with open(manifest_file, "w") as fp:
        json.dump(manifest, fp, indent=2, sort_keys=True)

    log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))

if __name__ == "__main__":
    main()


from selenium import webdriver
from selenium.webdriver.common.by import By

driver = webdriver.Chrome(executable_path="/Users/shockingjmh/opt/anaconda3/envs/py10/bin/chromedriver") # 다운받은 크롬드라이버 위치
driver.get("https://www.wadiz.kr/web/wreward/main?keyword=&endYn=ALL&order=recommend") #크롤링 대상 페이지
driver.maximize_window()
import time

from datetime import datetime

start_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
num = [1,2,3,4,5]

import random

try:
    print("for start")
    for i in range(2500):
        print(str(i+1) + " process")
        button = driver.find_element_by_xpath(f'//*[@id="main-app"]/div[2]/div/div[3]/div[2]/div[2]/div/button') #더보기버튼 xpath

        time.sleep(random.randint(1, 5) * 0.01)
        driver.execute_script("arguments[0].click();", button) #click()으로 에러가나서 써줌

except Exception as e:
    print("except : " + str(e))
    button = driver.find_element_by_class_name #더보기버튼

wadiz_title=[]
wadiz_url=[]

table = driver.find_element_by_class_name('ProjectCardList_container__3Y14k')
rows = table.find_elements_by_class_name('ProjectCardList_item__1owJa')

for index, value in enumerate(rows):
    title = value.find_element_by_class_name('CommonCard_title__1oKJY')
    wadiz_title.append(title.text)

    url_class = value.find_element_by_class_name('CommonCard_container__e_ebQ')
    url = url_class.get_attribute('href')

    wadiz_url.append(url)

import pandas as pd
import numpy as np
from datetime import datetime

df1 = pd.DataFrame({'title' : wadiz_title, 'url' : wadiz_url})
df1.to_csv("wadiz_title_url_"+ str(datetime.now().strftime('%Y%m%d%H%M%S')) +".csv",mode='w',encoding='utf-8-sig')

end_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

print("start time : " + str(start_time))
print("end time : " + str(end_time))


import caffe
import cv2
import sys

def deploy(img_path):

    caffe.set_mode_gpu()

    # Initialize the Caffe model using the model trained in DIGITS. Which two files constitute your trained model?
    net = caffe.Classifier('/dli/data/digits/20200325-142619-1525/deploy.prototxt', '/dli/data/digits/20200325-142619-1525/snapshot_iter_5400.caffemodel',
                           channel_swap=(2,1,0),
                           raw_scale=255,
                           image_dims=(256, 256))

    # Create an input that the network expects. This is different for each project, so don't worry about the exact steps, but find the dataset job directory to show you know that whatever preprocessing is done during training must also be done during deployment.
    input_image= caffe.io.load_image(img_path)
    input_image = cv2.resize(input_image, (256,256))
    mean_image = caffe.io.load_image('/dli/data/digits/20200325-141614-588c/mean.jpg')
    input_image = input_image-mean_image

    # Make prediction. What is the function and the input to the function needed to make a prediction?
    prediction = net.predict([input_image])  ##REPLACE WITH THE FUNCTION THAT RETURNS THE OUTPUT OF THE NETWORK##([##REPLACE WITH THE INPUT TO THE FUNCTION##])

    # Create an output that is useful to a user. What is the condition that should return "whale" vs. "not whale"?
    if prediction.argmax()==0: ##REPLACE WITH THE CONDITION THAT WOULD MAKE THE FUNCTION RETURN WHALE##:
        return "whale"
    else:
        return "not whale"

##Ignore this part
if __name__ == '__main__':
    print(deploy(sys.argv[1]))


'''
Created on Oct 4, 2014

@author: sunnycomes
'''

from tornado.web import RequestHandler

from src.common.post_parser import get_all_parsed_posts
from src.common.settings import get_site_info, get_3rd_party_snippets

class IndexHandler(RequestHandler):
    '''
    This handler is used to deal with the requests for index page.
    Visiting url is http://localhost:9999/
    '''
    def get(self):
        posts = get_all_parsed_posts()

        params = get_site_info()

        snippets = get_3rd_party_snippets()

        template_file_name = "index.html"

        self.render(template_file_name, posts = posts, params = params, snippets = snippets)

handler = (r"/", IndexHandler)


# Pre-requisite pip install Circlify and matplotlib in the terminal.
import csv
import circlify
import matplotlib.pyplot as plt

#Sample Data
data = [{'id': 'World', 'datum': 6964195249, 'children': [

              {'id': "North America", 'datum': 450448697, 'children':
                  [
                     {'id': "United States", 'datum': 308865000},
                     {'id': "Mexico", 'datum': 107550697},
                     {'id': "Canada", 'datum': 34033000}
                  ]},

              {'id': "South America", 'datum': 278095425, 'children':
                  [
                     {'id': "Brazil", 'datum': 192612000},
                     {'id': "Colombia", 'datum': 45349000},
                     {'id': "Argentina", 'datum': 40134425}
                  ]},

              {'id': "Europe", 'datum': 209246682, 'children':
                  [
                     {'id': "Germany", 'datum': 81757600},
                     {'id': "France", 'datum': 65447374},
                     {'id': "United Kingdom", 'datum': 62041708}
                  ]},

              {'id': "Africa", 'datum': 311929000, 'children':
                  [
                     {'id': "Nigeria", 'datum': 154729000},
                     {'id': "Ethiopia", 'datum': 79221000},
                     {'id': "Egypt", 'datum': 77979000}
                  ]},

              {'id': "Asia", 'datum': 2745929500, 'children':
                  [
                     {'id': "China", 'datum': 1336335000},
                     {'id': "India", 'datum': 1178225000},
                     {'id': "Indonesia", 'datum': 231369500}
              ]}

    ]}]

# Compute circle positions thanks to the circlify() function
# The maximum radius is set to 1.
circles = circlify.circlify(
    data,
    show_enclosure=False,
    target_enclosure=circlify.Circle(x=0, y=0, r=1)
)

# To ensure in Tableau the sizing works the fig size is the same x*y.
fig, ax = plt.subplots(figsize=(10,10))

# Add axes
ax.axis('on')

# Find axis boundaries, lim will be 1.0 with current settings
lim = max(
    max(
        abs(circle.x) + circle.r,
        abs(circle.y) + circle.r,
    )
    for circle in circles
)
plt.xlim(-lim, lim)
plt.ylim(-lim, lim)

# Amend Radius zone, Make sure Inner pad always > than Outer pad
padding_outer = 1
padding_inner = 1

header = ['ID', 'X co-ord', 'Y-cord', 'Radius', 'Rank']
rank = 1

with open('countries.csv', 'w') as f:
    writer = csv.writer(f)
    writer.writerow(header)

    # World Level
    for circle in circles:
        if circle.level != 1:
            continue
        x, y, r = circle
        label = circle.ex["id"]
        print(label,x,y,r)
        printdata = [label, x, y, r, rank]
        rank = rank + 1
        writer.writerow(printdata)

        # Not needed for Tableau
        ax.add_patch(plt.Circle((x, y), r*padding_outer, alpha=0.5, linewidth=2, color="lightblue"))

        # Continent Level
        for circle in circles:
            if circle.level != 2:
              continue
            x, y, r = circle
            label = circle.ex["id"]
            print(label,x,y,r)
            printdata = [label, x, y, r, rank]
            rank = rank + 1
            writer.writerow(printdata)

            # Not needed for Tableau
            ax.add_patch(plt.Circle((x, y), r*padding_outer, alpha=0.5, linewidth=2, color="lightblue"))

        # Country Level
        for circle in circles:
            if circle.level != 3:
              continue
            x, y, r = circle
            label = circle.ex["id"]
            print(label,x,y,r)
            printdata = [label, x, y, r, rank]
            rank = rank + 1
            writer.writerow(printdata)

            # Not needed for Tableau
            ax.add_patch(plt.Circle((x, y), r*padding_inner, alpha=0.5, linewidth=2, color="#69b3a2"))
            plt.annotate(label, (x,y), ha='center', color="white")

# Show The Example Of The Graph We Will Be Recreating!
plt.show()

2021 Blogs/2021-09-21 Don't Burst My Bubble/main.py


import os
import re

from rich import print
from rich.console import Console

MINT: str = "./cadence/transactions/mint-batch.cdc"

# Uncomment some of these to make the script work

# MAINNET
NETWORK: str = "mainnet"
FROM: str = "0x1f56a1e665826a52"
CONTRACT: str = "82b54037a8f180cf"
LOWEST_BLOCK = 19200000
# LOWEST_BLOCK = 19430500

# TESTNET
# NETWORK: str = "testnet"

def main():
    console = Console()

    # Get block height
    command = f"flow blocks get latest --network {NETWORK}"
    block_height = os.popen(command).read()
    block_height = re.search("Height\s*([0-9]*)", block_height)
    block_height = block_height.group(1)
    block_height = int(block_height)

    distributed = []

    with console.status(f"[bold green]Fetching down to flow block {LOWEST_BLOCK}"):
        while block_height > LOWEST_BLOCK:
            scan_blocks = 1000
            command = f"""
                flow events get \
                A.{CONTRACT}.Shard.Withdraw \
                --network {NETWORK} \
                --start {block_height - scan_blocks} \
                --end {block_height}
            """
            events = os.popen(command).read()
            console.log(f"Fetched blocks {block_height - scan_blocks}")

            withdrawals = re.findall("([0-9]*)\s*.*from.*(0x[a-f0-9]*)", events)

            for withdrawal in withdrawals:
                if FROM in withdrawal:
                    distributed.append(withdrawal)

            block_height -= scan_blocks

        console.print("[bold]Total Distibuted Shards:", len(distributed))

if __name__ == "__main__":
    main()


import os
import matplotlib
matplotlib.use('Agg') # No pictures displayed
import pylab
import librosa
import librosa.display
import numpy as np

os.chdir(r'G:\000. UCD- Data Science\Research Project\Demo_Songs\spec')

y, sr = librosa.load('pop.00000.au')
librosa.feature.melspectrogram(y=y, sr=sr)
# array([[  2.891e-07,   2.548e-03, ...,   8.116e-09,   5.633e-09],
# [  1.986e-07,   1.162e-02, ...,   9.332e-08,   6.716e-09],
# ...,
# [  3.668e-09,   2.029e-08, ...,   3.208e-09,   2.864e-09],
# [  2.561e-10,   2.096e-09, ...,   7.543e-10,   6.101e-10]])

# Using a pre-computed power spectrogram

D = np.abs(librosa.stft(y))**2
S = librosa.feature.melspectrogram(S=D)

# Passing through arguments to the Mel filters
S = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=128,
                                    fmax=8000)

#import matplotlib.pyplot as plt
plt.figure(figsize=(10, 4))
librosa.display.specshow(librosa.power_to_db(S,
                                             ref=np.max),
                         y_axis='mel', fmax=8000,
                         x_axis='time')
plt.colorbar(format='%+2.0f dB')
plt.title('POP')
plt.tight_layout()


"""
Cosmic Ray Window Functions
===========================
Compute the cosmic ray W window functions that give the heating distributions of cosmic rays in the simulation
"""

# Required modules and packages
from typing import Tuple, Callable
from scipy.integrate import quad
import cosmic_ray_general as crg
import cosmic_ray_coeffs as crc
import numpy as np
import math as mt
from numba import njit
from scipy.interpolate import interp1d
from scipy.io import savemat

# Accuracy parameters
INTERP_NUM_SAMPLES = 1_000  # When computing Ke_now(r) and W(r) how many samples should be used in interpolation
KE_R_LOW_KE_CUTOFF = 1e-6  # Lower limit on KE used in Ke_now(r) computation, in MeV
REFINE_REGION = 4  # Factor by which refined region is smaller than entire window function
REFINE_FACTOR = 2  # Factor by which refined region resolution is increased

# Geometric Functions
def distance_from_emitting_cell(length_in_cells: int, cell_side_length: float) -> np.ndarray:
    """ Compute the centre to centre distance of cells from the emission cell.

    Note this is not in PBC

    Parameters
    ----------
    length_in_cells : Int
        Number of cells along each direction the cube has, e.g. the cube has a total of length_in_cells**3 cells
    cell_side_length : Float
        Side length of an individual cell

    Returns
    -------
    Array
        length_in_cells x length_in_cells x length_in_cells array of distance each cells if from the emission cell,
        the floor(length_in_cells/2), floor(length_in_cells/2), floor(length_in_cells/2) th cell
    """
    # Setup datastructure for final result
    build_up_distances = np.zeros((length_in_cells, length_in_cells, length_in_cells))

    # Get 1D periodic distance squared
    one_d_idx = np.arange(length_in_cells)
    ems_c_idx = mt.floor(length_in_cells/2)
    one_d_p_sq_distance = (one_d_idx - ems_c_idx)**2

    # Sum over hyper-planes building up distance
    build_up_distances += one_d_p_sq_distance[:, np.newaxis, np.newaxis]
    build_up_distances += one_d_p_sq_distance[np.newaxis, :, np.newaxis]
    build_up_distances += one_d_p_sq_distance[np.newaxis, np.newaxis, :]

    # Convert to distance in cell_side_length units
    return cell_side_length*np.sqrt(build_up_distances)

# Proton evolution functions
@njit(cache=True)
def range_of_surviving_ke(z_now: float, z_ini: float, ke_min: float, ke_max: float) -> Tuple[float, float]:
    """ Compute the minimum and maximum kinetic energy of CR protons that have not been absorbed by z_now emitted
    at z_ini

    Parameters
    ----------
    z_now : float
        Current redshift
    z_ini : float
        Redshift of emission
    ke_min : float
        Minimum kinetic energy of CR protons injected into the IGM
    ke_max : float
        Maximum kinetic energy of CR protons injected into the IGM

    Returns
    -------
    Float
        Minimum kinetic energy of surviving CR protons
    Float
        Maximum kinetic energy of surviving CR protons (if 0 all CR protons are absorbed)
    """

    # Check for case that all CR protons were absorbed
    ke_ems_for_protons_being_absorbed = crg.initial_ke(0, z_now, z_ini)
    if ke_ems_for_protons_being_absorbed >= ke_max:
        return 0.0, 0.0

    # Determine if any CR protons have been absorbed and hence the minimum surviving ke
    if ke_ems_for_protons_being_absorbed >= ke_min:
        min_surviving_ke = 0
    else:
        min_surviving_ke = crg.current_ke(ke_min, z_now, z_ini)

    # Maximum surviving ke can be determined similarly
    return min_surviving_ke, crg.current_ke(ke_max, z_now, z_ini)

def comoving_path_proton_for_ke_now(z_now: float, ke_now: np.ndarray, z_ini: float, smoothing: bool = False)\
        -> np.ndarray:
    """ Comoving path length a CR proton would travel between z_0 and z

    Note this is a wrapper of comoving_path_proton but uses ke_now rather than ke_ini as an input

    Parameters
    ----------
    z_now : Float
        Redshift proton is at now
    ke_now : Array
        Kinetic energy of the protons at z_now in MeV
    z_ini : Float
        Source redshift of protons
    smoothing : Boolean
        Whether to smooth Delta R around the transition kinetic energy

    Returns
    -------
    Array
        Comoving path lengths travelled in comoving Megaparsec
    """

    # Function parameters
    smoothing_fraction = 0.5  # Fraction below KE_REL_TRANS_MEV to which smoothing is applied

    # Compute initial KE and path length without any smoothing
    v_initial_ke = np.vectorize(crg.initial_ke)
    ke_ini = v_initial_ke(ke_now, z_now, z_ini)
    path_lengths = crg.comoving_path_proton(z_now, ke_ini, z_ini)

    # If not smoothing then simply return the non-smoothed path lengths
    if not smoothing:
        return path_lengths

    # Add smoothing to value below the relativistic kinetic energy transition
    affected_idxs = np.logical_and(ke_now < crg.KE_REL_TRANS_MEV, ke_now > smoothing_fraction*crg.KE_REL_TRANS_MEV)

    frac_rel_only = (np.log(ke_now[affected_idxs]) - np.log(smoothing_fraction*crg.KE_REL_TRANS_MEV))\
        / (-np.log(smoothing_fraction))  # Smoothing performed linearly in log(KE_now)
    frac_transition = 1 - frac_rel_only

    path_lengths[affected_idxs] = frac_transition*path_lengths[affected_idxs] + \
        frac_rel_only*crg.comoving_path_proton(z_now, ke_ini[affected_idxs], z_ini, True)

    return path_lengths

def ke_now_given_r_func(z_now: float, z_ini: float, ke_now_min: float, ke_now_max: float) -> Callable:
    """ Return a function to compute ke_now given the path-length (r), i.e. T_r from Notes

    Parameters
    ----------
    z_now : Float
        Redshift of heating
    z_ini : Float
        Redshift of CR emission
    ke_now_min : Float
        Minimum kinetic energy of surviving protons
    ke_now_max
        Maximum kinetic energy of surviving protons

    Returns
    -------
    Callable
        Function to compute ke_now given r
    """

    # Compute R(ke_now) first
    if ke_now_max < KE_R_LOW_KE_CUTOFF:
        ke_nows = np.linspace(ke_now_min, ke_now_max, INTERP_NUM_SAMPLES)
    else:
        ke_nows = np.logspace(np.log10(max(ke_now_min, KE_R_LOW_KE_CUTOFF)),
                              np.log10(ke_now_max), INTERP_NUM_SAMPLES)
    r_for_ke = comoving_path_proton_for_ke_now(z_now, ke_nows, z_ini, smoothing=True)

    # Invert via interpolation (implicit monotonicity assumption)
    interp_func = interp1d(np.log(r_for_ke), np.log(ke_nows), kind='linear', bounds_error=False, fill_value=np.NAN,
                           assume_sorted=False)
    return lambda r: np.exp(interp_func(np.log(r)))  # Log-log interpolation used for improved accuracy

# Free Streaming window function computation
def free_stream_w_given_r_func(z_now: float, z_ini: float, alpha: float, ke_ini_min: float, ke_ini_max: float,
                               func_ke_now_given_r: Callable, cr_coeff: float) -> Callable:
    """ Return a function to compute the free streaming window function value given the cell distance r

    Parameters
    ----------
    z_now : Float
        Redshift at which heating is taking place
    z_ini : Float
        Redshift of emission
    alpha : Float
        Exponent of the power-law spectrum of CRs
    ke_ini_min : Float
        Minimum KE cutoff of the power-law in MeV
    ke_ini_max : Float
        Maximum KE cutoff of the power-law in MeV
    func_ke_now_given_r : Callable
        Function which computes ke_now given a travelled comoving path length r
    cr_coeff : Float
        Cosmic ray heating coefficient that has already been computed for this z_now, z_ini pair

    Returns
    -------
    Callable
        Function to compute the free-streaming window function at a given r via interpolation
    """

    # Find r values to sample at and the corresponding ke_now (sample in log delta r for improved interp accuracy)
    ke_now_min, ke_now_max = range_of_surviving_ke(z_now, z_ini, ke_ini_min, ke_ini_max)
    r_min, r_max = comoving_path_proton_for_ke_now(z_now, np.array([ke_now_min, ke_now_max]), z_ini, smoothing=True)

    delta_rs = np.logspace(np.log10(r_min*1e-6), np.log10(r_max*(1-1e-6) - r_min), INTERP_NUM_SAMPLES)
    rs = r_min + delta_rs
    ke_nows = func_ke_now_given_r(rs)

    # Compute the r' derivative term due to having delta function shells
    dke = np.maximum(ke_nows*1e-6, 1e-5)
    r_primes = (comoving_path_proton_for_ke_now(z_now, ke_nows + dke, z_ini, smoothing=True) -
                comoving_path_proton_for_ke_now(z_now, ke_nows, z_ini, smoothing=True)) / dke

    # Compute the window function at the chosen rs
    integrand_term = np.array([crc.cosmic_ray_c_integrand(ke_now, z_now, z_ini, alpha, ke_ini_min, ke_ini_max) for
                               ke_now in ke_nows])
    ws = integrand_term / (4 * cr_coeff * np.pi * (rs**2) * np.abs(r_primes))

    # Window function interpolator
    w_interp = interp1d(np.log(rs), np.log(ws), kind='linear', bounds_error=False, fill_value=-np.infty)
    return lambda r: np.exp(w_interp(np.log(r)))  # Log-log interpolation used for improved accuracy

def free_stream_window_function(z_now: float, z_ini: float, alpha: float, ke_ini_min: float, ke_ini_max: float,
                                length_in_cells: int, cell_side_length: float, cr_coeff: float) -> np.ndarray:
    """ Compute the free-streaming window function for emission at z_ini and heating at z_now

    Parameters
    ----------
    z_now : Float
        Redshift at which heating is taking place
    z_ini : Float
        Redshift of emission
    alpha : Float
        Exponent of the power-law spectrum of CRs
    ke_ini_min : Float
        Minimum KE cutoff of the power-law in MeV
    ke_ini_max : Float
        Maximum KE cutoff of the power-law in MeV
    length_in_cells : int
        Number of cells along each direction the simulation cube has, e.g. the cube is length_in_cells**3 cells total
    cell_side_length : Float
        Side length of an individual cell in cMpc
    cr_coeff : Float
        Cosmic ray heating coefficient that has been pre-computed for this z_now, z_ini pair

    Returns
    -------
    Array
        length_in_cells x length_in_cells x length_in_cells array representing the window function in real-space
    """

    # Check for valid z_now and z_ini relationship, and trivial cases
    if z_now > z_ini:
        raise ValueError(f'z_now ({z_now}) must be less than z_ini ({z_ini})')

    if z_now == z_ini:  # Causality dictates all heating is in origin cell
        window_func = np.zeros((length_in_cells, length_in_cells, length_in_cells))
        window_func[0, 0, 0] = 1
        return window_func

    ke_now_min, ke_now_max = range_of_surviving_ke(z_now, z_ini, ke_ini_min, ke_ini_max)
    if ke_now_max == 0:  # All protons absorbed so window function is ill-defined
        window_func = np.zeros((length_in_cells, length_in_cells, length_in_cells))
        return window_func

    # Find minimum and maximum distance a proton can be at
    r_min, r_max = comoving_path_proton_for_ke_now(z_now, np.array([ke_now_min, ke_now_max]), z_ini, smoothing=True)
    sim_box_border_r = cell_side_length*mt.ceil(length_in_cells/2 - 1)

    if r_min >= sim_box_border_r:  # If r_min exceeds distance to border then return a uniform window function
        return np.ones((length_in_cells, length_in_cells, length_in_cells)) / length_in_cells**3

    if r_max < cell_side_length/REFINE_FACTOR:  # In current model not travelled far enough for any to escape the cell
        window_func = np.zeros((length_in_cells, length_in_cells, length_in_cells))
        window_func[0, 0, 0] = 1
        return window_func

    # NOTE: until the fftshift back we are not working PBC from this point
    # Initialize ke_now(r) and window function interpolators and datastructure
    func_ke_now_r = ke_now_given_r_func(z_now, z_ini, ke_now_min, ke_now_max)
    func_w_r = free_stream_w_given_r_func(z_now, z_ini, alpha, ke_ini_min, ke_ini_max, func_ke_now_r, cr_coeff)
    w = np.zeros((length_in_cells, length_in_cells, length_in_cells))

    # Find distances of cells from the emitting cell
    rs = distance_from_emitting_cell(length_in_cells, cell_side_length)
    origin_position = mt.floor(length_in_cells / 2)
    rs[origin_position, origin_position, origin_position] = cell_side_length/4  # Very simple handling of origin

    # Compute window function at locations that are inside the simulation cube at normal resolution
    valid_rs = np.logical_and(rs <= min(sim_box_border_r, r_max), rs >= r_min)
    w[valid_rs] = func_w_r(rs[valid_rs])*(cell_side_length**3)  # Set the window function for that region

    # Calculate window function on the refined grid if required
    if r_min < mt.floor(length_in_cells / REFINE_REGION)*cell_side_length:
        # Get refined region size (forced to odd for isotropy) and location at original resolution
        refined_region_or_cell_size = mt.floor(length_in_cells / REFINE_REGION)
        refined_region_or_cell_size = refined_region_or_cell_size - np.mod(refined_region_or_cell_size, 2)
        refine_region = slice(mt.floor(length_in_cells/2)-mt.floor(refined_region_or_cell_size/2),
                              mt.floor(length_in_cells/2)+mt.ceil(refined_region_or_cell_size/2))

        # Get refined region size at increased resolution, +2 -2 is padding for convolution
        refined_region_rr_cell_size = (refined_region_or_cell_size+2)*REFINE_FACTOR - 2

        # Compute the window function on the refined grid
        refined_rs = distance_from_emitting_cell(refined_region_rr_cell_size, cell_side_length/REFINE_FACTOR)
        valid_refined_rs = np.logical_and(refined_rs <= r_max, refined_rs >= r_min)
        w_refined = np.zeros((refined_region_rr_cell_size, refined_region_rr_cell_size, refined_region_rr_cell_size))
        w_refined[valid_refined_rs] = func_w_r(refined_rs[valid_refined_rs])*((cell_side_length/REFINE_FACTOR)**3)

        # Convolve over possible source locations in the refined grid
        source_window_function = np.zeros(w_refined.shape)
        source_window_function[:REFINE_FACTOR, :REFINE_FACTOR, :REFINE_FACTOR] = 1/(REFINE_FACTOR**3)
        w_refined = np.real(np.fft.ifftn(np.fft.fftn(source_window_function)*np.fft.fftn(w_refined)))

        # Strip padding and coarsen
        slice_without_padding = slice(REFINE_FACTOR - 1, refined_region_rr_cell_size - REFINE_FACTOR + 1)
        w_refined = w_refined[slice_without_padding, slice_without_padding, slice_without_padding]
        w[refine_region, refine_region, refine_region] = 0
        for offset_x in range(REFINE_FACTOR):
            for offset_y in range(REFINE_FACTOR):
                for offset_z in range(REFINE_FACTOR):
                    w[refine_region, refine_region, refine_region] += \
                        w_refined[offset_x::REFINE_FACTOR, offset_y::REFINE_FACTOR, offset_z::REFINE_FACTOR]

    # For compression efficiency set any coefficient that is very small to 0. These tiny coefficients can occur at very
    # large distances and due to numerical noise from the convolution in the refinement step
    w[np.isclose(w, 0, atol=1e-10)] = 0

    # Uniform component if CRs reach outside the simulation box
    if r_max > sim_box_border_r:
        ke_escape = func_ke_now_r(sim_box_border_r)
        partial_coeff = quad(crc.cosmic_ray_c_integrand, ke_escape, np.inf,
                             args=(z_now, z_ini, alpha, ke_ini_min, ke_ini_max), epsabs=0, epsrel=1e-3, limit=100)[0]
        w = w + partial_coeff/(cr_coeff * (length_in_cells**3))

    # Shift back into PBC (as per MATLAB sim convention)
    return np.fft.fftshift(w)

# Store window function
def save_window_function(window_function: np.ndarray, file_name: str):
    """ Store the given window function in the format required by 21cmSim

    Parameters
    ----------
    window_function: Array
        The window function to be saved to disk
    file_name : String
        File to save the window function to
    """
    # Normalize
    window_function = window_function/np.sum(window_function)

    # Store compressed in fourier space
    fourier_window = np.real(np.fft.fftn(window_function))  # Only real needed as window functions are Isotropic
    fourier_window = fourier_window.astype(np.single)
    savemat(file_name, {'w': fourier_window}, do_compression=True)


import re

from setuptools import setup, find_packages

with open("readme.md", "r") as fh:
    long_description = fh.read()

_version_regex = (
    r"^__version__ = ('|\")((?:[0-9]+\.)*[0-9]+(?:\.?([a-z]+)(?:\.?[0-9])?)?)\1$"
)

try:
    with open("antispam/__init__.py") as stream:
        match = re.search(_version_regex, stream.read(), re.MULTILINE)
        version = match.group(2)
except FileNotFoundError:
    version = "0.0.0"

def parse_requirements_file(path):
    with open(path) as fp:
        dependencies = (d.strip() for d in fp.read().split("\n") if d.strip())
        return [d for d in dependencies if not d.startswith("#")]

setup(
    name="Discord Anti-Spam",
    version=version,
    author="Skelmis",
    author_email="ethan@koldfusion.xyz",
    description="An easy to use package for anti-spam features in python discord libraries.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    project_urls={
        "Support server": "https://discord.gg/BqPNSH2jPg",
        "Issue tracker": "https://github.com/Skelmis/DPY-Anti-Spam/issues",
        "Documentation": "https://dpy-anti-spam.readthedocs.io/en/latest",
        "Homepage": "https://github.com/Skelmis/DPY-Anti-Spam",
    },
    packages=find_packages(include=("antispam", "antispam.*")),
    install_requires=parse_requirements_file("requirements.txt"),
    extras_requires={
        "dev": parse_requirements_file("dev-requirements.txt"),
        "mongo": ["motor", "dnspython", "pytz"],
        "redis": ["redis", "orjson", "hiredis"],
    },
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
        "Development Status :: 5 - Production/Stable",
    ],
    python_requires=">=3.8",
)


from django.shortcuts import render
from django.core.mail import send_mail
from .models import opentime
def responsemail(tomail,name):
    send_mail(
            'From Dentrist website  ',
            'Hello, '+name+' Thanks for using oru website. \nOur Doctor will respond soon',
            '8921test@gmail.com',
            [tomail],
            fail_silently=False,)
def response(request):
    if request.method=='POST':
        name=request.POST['name']
        mail=request.POST['mail']
        date=request.POST['date']
        time=request.POST['time']
        send_mail(
            'From Dentrist website  ',
            'Hello, '+name+' Thanks for using oru website. \nYour Appoinment has been confirmed.  You can vist the hospital on the specifed time.',
            '8921test@gmail.com',
            [mail],
            fail_silently=False,)
        send_mail(
            'Appoinment from '+name,
            name+'Has made an appoinment on :'+date+' in :'+time+'\n\n His message was: '+'\n\nHis',
            mail,
            ['sudharsansriram8921@gmail.com'],
            fail_silently=True,)
        print('hi')
        return render(request,'response.html',{'name':name})
    else:
        return render(request,'response.html')

def index(request):
    availablity = opentime.objects.all()
    context = { 'availablity': availablity,}
    return render(request,'index.html',context)

def about(request):
    return render(request,'about.html')

def blog(request):
    return render(request,'blog.html')

def contact(request):
    if request.method=='POST':
        name=request.POST['name']
        mail=request.POST['mail']
        subject=request.POST['subject']
        message=request.POST['message']
        responsemail(mail,name)
        send_mail(
            'message from '+name+' regarding '+subject,
            message+'\n\n his mail: '+mail,
            mail,
            ['sudharsansriram8921@gmail.com'],
            fail_silently=False,)
        return render(request,'contact.html',{'name':name,'mail':mail})
    else:
        return render(request,'contact.html')

def doctors(request):
    return render(request,'doctors.html')

def services(request):
    return render(request,'services.html')

def login(request):
    return render(request,'login.html')


import sys
import traceback
from PyQt5.QtWidgets import QWidget, QApplication, QVBoxLayout, QTreeWidget, QTreeWidgetItem, QProxyStyle, \
    QStyleOptionViewItem

class YqStyle(QProxyStyle):
    def drawControl(self, QStyle_ControlElement, QStyleOption, QPainter, widget=None):
        super().drawControl(QStyle_ControlElement, QStyleOption, QPainter, widget)

        # print(QStyle_ControlElement)

        # 这里的画是按照一行一画的
        if isinstance(QStyleOption, QStyleOptionViewItem):
            item = widget.itemFromIndex(QStyleOption.index)
            for i in range(item.columnCount()):
                print(item.text(i))

class MyWidget(QWidget):
    def __init__(self):
        super().__init__()
        self.init_ui()

    def init_ui(self):
        vbox = QVBoxLayout()
        treewidget = QTreeWidget()
        vbox.addWidget(treewidget)

        self.setLayout(vbox)

        treewidget.setColumnCount(3)

        item1 = QTreeWidgetItem(['1', '2', '3'])
        treewidget.addTopLevelItem(item1)

        item2 = QTreeWidgetItem(['1', '2', '3'])
        item1.addChild(item2)

        treewidget.setStyle(YqStyle())

def main():
    sys.excepthook = traceback.print_exception
    app = QApplication(sys.argv)
    widget = MyWidget()
    widget.show()
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()


import operator
import pickle
from collections import defaultdict
from multiprocessing import Pool

import numpy as np
from scipy.spatial.distance import euclidean

# from speech_dtw import _dtw

def lol(args):
    i, x_test, train_sequences, cost_mat, test_expected_labels = args
    costs = defaultdict(int)
    for label in train_sequences.keys():
        for x_train in train_sequences[label]:
            path, cost = _dtw.multivariate_dtw(x_test, x_train,
                                               metric='euclidean')
            cost_mat[i, label] += cost
        cost_mat[i, label] /= len(train_sequences[label]) # normalize
    print(i, test_expected_labels[i], cost_mat[i])

def main():

    with open('train_sequences', 'rb') as f:
        train_sequences = pickle.load(f)
    with open('test_sequences', 'rb') as f:
        test_sequences = pickle.load(f)
    with open('test_expected_labels', 'rb') as f:
        test_expected_labels = pickle.load(f)

    for i, x_test in enumerate(test_sequences):
        test_sequences[i] = np.asarray(
            np.array([list(x_test)], dtype=np.double).T, order='c')

    for label in train_sequences.keys():
        for i, x_train in enumerate(train_sequences[label]):
            train_sequences[label][i] = np.asarray(
                np.array([list(x_train)], dtype=np.double).T, order='c')

    label_set = list(train_sequences.keys())

    cost_mat = np.ndarray(shape=(len(test_sequences), len(label_set)))

    pool = Pool()

    mapped = [(i, x_test, train_sequences, cost_mat, test_expected_labels) \
              for i, x_test in enumerate(test_sequences)]

    pool.map(lol, mapped)
    pool.close()

    pool.join()

    with open('cost_mat.dat', 'wb') as f: pickle.dump(cost_mat, f)

def score():

    with open('test_expected_labels', 'rb') as f:
        expected_labels = pickle.load(f)

    with open('cost_mat.dat', 'rb') as f:
        cost_mat = pickle.load(f)

    predicted_labels = np.argmin(cost_mat, axis=1)

    precision = np.mean(predicted_labels == expected_labels)

    return 1 - precision

if __name__ == '__main__':
    print(score())


import collections
import itertools
import numpy as np
import tensorflow as tf
import sklearn
import tensorflow_models
import logging
import local_models.loggin

logger = logging.getLogger(__name__)

#################### Util functions ####################

class fuckit(object):
    def __init__(self, message=None):
        self.message = message
    def __enter__(self): return self
    def __exit__(self, *args):
        if self.message is not None:
            print(self.message)
        return True

def build_dataset(sentences, vocabulary_size=50000):
    '''
    Build the dictionary and replace rare words with UNK token.

    Parameters
    ----------
    sentences: list of list of tokens
    vocabulary_size: maximum number of top occurring tokens to produce,
        rare tokens will be replaced by 'UNK'
    '''
    words = itertools.chain(*sentences)
    count = collections.Counter()
    total = 0
    for word in words:
        count[word] += 1
        total += 1
    vocab = dict(count.most_common(vocabulary_size-1))

    word_index_mapping = {word: i+1 for i, word in enumerate(vocab)} # {word: index}
    word_index_mapping["UNK"] = 0

    data = []
    k = 0
    for i, sentence in enumerate(sentences):
        data.append([])
        for j, word in enumerate(sentence):
            if word in word_index_mapping:
                data[-1].append(word_index_mapping[word])
            else:
                data[-1].append(word_index_mapping["UNK"])
                count["UNK"] += 1
    reverse_mapping = dict(zip(word_index_mapping.values(), word_index_mapping.keys()))
    return data, count, word_index_mapping, reverse_mapping

def build_dictionary(sentences, vocabulary_size=50000):
    '''
    Build the dictionary and replace rare words with UNK token.

    Parameters
    ----------
    sentences: list of list of tokens
    vocabulary_size: maximum number of top occurring tokens to produce,
        rare tokens will be replaced by 'UNK'
    '''
    words = itertools.chain(*sentences)
    count = collections.Counter()
    total = 0
    for word in words:
        count[word] += 1
        total += 1
    vocab = dict(count.most_common(vocabulary_size-1))

    word_index_mapping = {word: i+1 for i, word in enumerate(vocab)} # {word: index}
    word_index_mapping["UNK"] = 0

    reverse_mapping = dict(zip(word_index_mapping.values(), word_index_mapping.keys()))
    return word_index_mapping, reverse_mapping

def build_dataset_predictionary(sentences, dictionary):
    '''
    Build the dictionary and replace rare words with UNK token.

    Parameters
    ----------
    sentences: list of list of tokens
    vocabulary_size: maximum number of top occurring tokens to produce,
        rare tokens will be replaced by 'UNK'
    '''
    words = itertools.chain(*sentences)

    data = []
    k = 0
    for i, sentence in enumerate(sentences):
        data.append([])
        for j, word in enumerate(sentence):
            if word in dictionary:
                data[-1].append(dictionary[word])
            else:
                data[-1].append(dictionary["UNK"])
    return data

def generate_batch_skipgram(data, batch_size, num_skips, skip_window):
    '''
    Batch generator for skip-gram model.

    Parameters
    ----------
    data: list of list of words (or index of words)
    batch_size: number of words in each mini-batch
    num_skips: number of surrounding words on either direction (1: one word ahead and one word following)
    skip_window: number of words at both ends of a sentence to skip (1: skip the first and last word of a sentence)
    '''
    assert num_skips <= skip_window
    sentence_index = 0
    data_index = 0
    word_index = skip_window
    target_index = -num_skips
    while True:
        batch = np.ndarray(shape=(batch_size,1,2), dtype=np.int32)
        labels = np.ndarray(shape=(batch_size,2), dtype=np.int32)
        i = 0
        while i < batch_size:
            batch[i,0] = sentence_index, word_index
            labels[i] = sentence_index, word_index + target_index
            target_index += 1
            if word_index + target_index == min(word_index + num_skips + 1, len(data[sentence_index])): #end of num_skips window... move onto next center word
                target_index = -num_skips
                word_index += 1
                if word_index >= len(data[sentence_index]) - skip_window: #increment the sentence
                    sentence_index = (sentence_index + 1) % len(data)
                    word_index = skip_window
            else: #move to the next target, skipping 0
                target_index += 1 + (not target_index)

        yield batch, labels

def generate_batch_cbow(data, batch_size, num_skips, skip_window):
    '''
    Batch generator for CBOW (Continuous Bag of Words).
    batch should be a shape of (batch_size, num_skips)

    Parameters
    ----------
    data: list of list of words (or index of words)
    batch_size: number of words in each mini-batch
    num_skips: number of surrounding words in either direction (1: one word ahead and one word following)
    skip_window: number of words at both ends of a sentence to skip (1: skip the first and last word of a sentence)
    '''

    assert num_skips <= skip_window
    sentence_index = 0
    word_index = skip_window
    target_window = np.concatenate((np.arange(-num_skips, 0), np.arange(1, num_skips+1)))
    while True:
        batch = np.ndarray(shape=(batch_size,2*num_skips,2), dtype=np.int32) #batch, which_skiptarget, (sentence_index, word_index)
        labels = np.ndarray(shape=(batch_size,2), dtype=np.int32)
        for i in range(batch_size):
            batch[i,:,0] = sentence_index
            batch[i,:,1] = target_window + word_index
            labels[i] = sentence_index, word_index

            word_index += 1
            if word_index >= len(data[sentence_index]) - skip_window: #increment the sentence
                sentence_index = (sentence_index + 1) % len(data)
                while len(data[sentence_index]) < 2*num_skips + 1: #skip sentences that are too short
                    sentence_index = (sentence_index + 1) % len(data)
                word_index = skip_window

        yield batch, labels

def sentenceindex_2_word_list(sentences, word_indices):
    x = np.zeros(word_indices.shape[:-1])
    for x_indices in itertools.product(*map(range, x.shape)):
        try:
            s_i, w_i = word_indices[x_indices]
            x[x_indices] = sentences[s_i][w_i]
        except Exception as e:
            print(len(sentences), len(sentences[s_i]), s_i, w_i)
            raise e
    return x

class Word2Vec(sklearn.base.BaseEstimator):

    def __init__(self, batch_size=128, num_skips=2, skip_window=2,
        architecture='cbow', embedding_size=128, vocabulary_size=5000,
        loss_type='sampled_softmax_loss', n_neg_samples=64,
        optimizer="AdagradOptimizer", log_epochs=True,
        learning_rate=1.0, epochs=10000,
        #fitted params for cloning and pickling
        dictionary=None, reverse_dictionary=None, final_embeddings=None, beta0=None, model=None,
        trainable_embeddings=True, trainable_weights=True, trainable_bias=True):
        import tensorflow

        # bind params to class
        self.batch_size = batch_size
        self.num_skips = num_skips
        self.skip_window = skip_window
        self.architecture = architecture
        self.embedding_size = embedding_size
        self.vocabulary_size = vocabulary_size
        self.loss_type = loss_type
        self.n_neg_samples = n_neg_samples
        self.optimizer = optimizer
        self.learning_rate = learning_rate
        self.epochs = epochs
        self.log_epochs=log_epochs
        self.model=model
        self.trainable_embeddings=trainable_embeddings
        self.trainable_weights=trainable_weights
        self.trainable_bias=trainable_bias

        self.dictionary=dictionary
        self.reverse_dictionary=reverse_dictionary
        self.final_embeddings=final_embeddings
        self.beta0 = beta0

        # choose a batch_generator function for feed_dict
        self._choose_batch_generator()

        super().__init__()

    def __setstate__(self, d):
        self.__dict__.update(d)
        #self.model = None
        self.build_model()

    def __getstate__(self):
        with fuckit(): self.sess.close()
        with fuckit(): del self.sess
        with fuckit():
            with self.model.graph_.as_default():
                tf.reset_default_graph()
        with fuckit(): del self.model
        return self.__dict__

    def init_vars_(self, beta0=None):

        if beta0 is not None:
            embeddings, weights, biases = (
                beta0[:self.vocabulary_size*self.embedding_size].reshape(
                    self.vocabulary_size, self.embedding_size),
                beta0[self.vocabulary_size*self.embedding_size:2*self.vocabulary_size*self.embedding_size].reshape(
                    self.vocabulary_size, self.embedding_size),
                beta0[2*self.vocabulary_size*self.embedding_size:])
            self.sess.run(
                (self.model.embeddings_feed,
                 self.model.weights_feed,
                 self.model.biases_feed),
                feed_dict = {self.model.embeddings_ph: embeddings,
                    self.model.weights_ph: weights,
                    self.model.biases_ph: biases})

    def _choose_batch_generator(self):
        if self.architecture == 'skip-gram':
            self.generate_batch = generate_batch_skipgram
        elif self.architecture == 'cbow':
            self.generate_batch = generate_batch_cbow

    def build_model_(self):
        '''
        Init a tensorflow Graph containing:
        input data, variables, model, loss function, optimizer
        '''
        with tf.name_scope("w2v"):

            with tf.name_scope("input"):

                if self.architecture == 'skip-gram':
                    self.model.X = tf.placeholder(tf.int32, shape=[None,1])
                elif self.architecture == 'cbow':
                    self.model.X = tf.placeholder(tf.int32, shape=[None, 2*self.num_skips])
                self.model.y = tf.placeholder(tf.int32, shape=[None,1])
                self.model.sample_weight = tf.placeholder(tf.float32, shape=[None])

            with tf.name_scope("variables"):

                self.model.embeddings = tf.Variable(
                    tf.random_uniform([self.vocabulary_size, self.embedding_size], -1.0, 1.0), name="embeddings")

                self.model.weights = tf.Variable(
                    tf.truncated_normal([self.vocabulary_size, self.embedding_size],
                        stddev=1.0 / np.sqrt(self.embedding_size)), name="weights")

                self.model.biases = tf.Variable(tf.zeros([self.vocabulary_size]), name="biases")

                self.model.embeddings_ph = tf.placeholder(self.model.embeddings.dtype, shape=self.model.embeddings.shape)
                self.model.weights_ph = tf.placeholder(self.model.weights.dtype, shape=self.model.weights.shape)
                self.model.biases_ph = tf.placeholder(self.model.biases.dtype, shape=self.model.biases.shape)

                self.model.embeddings_feed = self.model.embeddings.assign(self.model.embeddings_ph)
                self.model.weights_feed = self.model.weights.assign(self.model.weights_ph)
                self.model.biases_feed = self.model.biases.assign(self.model.biases_ph)

            with tf.name_scope("embedding_layer"):
                embed = tf.nn.embedding_lookup(self.model.embeddings, self.model.X)
                self.model.embed = tf.reduce_sum(embed, axis=1)

            with tf.name_scope("loss"):
                if self.loss_type == "softmax_loss":
                    logits = tf.matmul(self.model.embed, tf.transpose(self.model.weights))
                    logits = tf.nn.bias_add(logits, self.model.biases)
                    labels_one_hot = tf.one_hot(self.model.y, self.vocabulary_size)
                    loss = tf.nn.softmax_cross_entropy_with_logits(
                        labels=labels_one_hot,
                        logits=logits)
                if self.loss_type == 'sampled_softmax_loss':
                    loss = tf.nn.sampled_softmax_loss(self.model.weights, self.model.biases, self.model.y,
                        self.model.embed, self.n_neg_samples, self.vocabulary_size)
                elif self.loss_type == 'nce_loss':
                    loss = tf.nn.nce_loss(self.model.weights, self.model.biases, self.model.y,
                        self.model.embed, self.n_neg_samples, self.vocabulary_size)
                self.model.loss = tf.reduce_mean(loss*self.model.sample_weight)

            with tf.name_scope("optimization"):
                self.model.optimizer = getattr(tf.train, self.optimizer)(self.learning_rate, name="optimizer")
                self.model.grads = self.model.optimizer.compute_gradients(self.model.loss)
                self.model.grad_application = self.model.optimizer.apply_gradients(self.model.grads)
                with tf.control_dependencies([self.model.grad_application]):
                    self.model.train_step = tf.no_op(name="train_step")

            # init op
            self.model.initializer = tf.initialize_all_variables()

    def get_global_sess(self):
        global TFGlobalSesh
        try:
            TFGlobalSesh
        except:
            TFGlobalSesh = tf.Session(graph = self.model.graph_)
        self.sess = TFGlobalSesh
        return self.sess

    def get_global_graph(self):
        global TFGlobalGraph
        try:
            TFGlobalGraph
        except:
            TFGlobalGraph = tf.Graph()
        self.model.graph_ = TFGlobalGraph
        return self.model.graph_

    def build_model(self):
        ''' Initializes a new graph, and then calls the .build_model_ method, which must be implemented by a TFEstimator '''
        self.model = tensorflow_models.NameSpace()
        self.get_global_graph()
        with self.model.graph_.as_default():
            self.build_model_()

    def _build_dictionaries(self, sentences):
        '''
        Process tokens and build dictionaries mapping between tokens and
        their indices. Also generate token count and bind these to self.
        '''

        data, count, dictionary, reverse_dictionary = build_dataset(sentences,
            self.vocabulary_size)
        try:
            assert len(dictionary) >= self.vocabulary_size
        except Exception as e:
            print("number of available words must be >= self.vocabulary_size")
            raise e
        self.dictionary = dictionary
        self.reverse_dictionary = reverse_dictionary
        self.count = count
        return data

    def fit(self, X, y=None, sample_weight=None, beta0=None,**fit_params):
        ''' Puts inputs into np format, initializes a session, builds the graph and then calls `.fit_`
                which can be overridden by individual models '''

        beta0 = beta0 if beta0 is not None else self.beta0
        if self.model is None:
            logger.info("building model!")
            self.build_model()
        if self.dictionary is None:
            self.dictionary, self.reverse_dictionary = build_dictionary(X, self.vocab_size)
            assert len(dictionary) == self.vocabulary_size
        data = build_dataset_predictionary(X, self.dictionary)

        with self.model.graph_.as_default():
            with tf.Session(graph = self.model.graph_) as self.sess:
                self.sess.run(self.model.initializer)
                self.init_vars_(beta0)
                fitted_model = self.fit_(data, sample_weight=sample_weight, **fit_params)
                embeddings, weights, biases = self.sess.run(
                    (self.model.embeddings, self.model.weights, self.model.biases))
                self.beta0 = np.concatenate([x.flatten() for x in
                    [embeddings, weights, biases]])
                self.embeddings = embeddings
                self.final_embeddings = embeddings / np.linalg.norm(embeddings)

        return fitted_model

    def fit_(self, X, sample_weight=None, feed_dict_extras={}):
        ''' Trains for a number of epochs.  Model input must be in self.model.X, output in self.model.y, loss in self.model.loss, and training using self.model.train_step '''
        # sample_weight is *PER SENTENCE*

        logger.info("fitting w2v model")
        epoch_log_str = "epoch: {:06d} ::: loss: {:.02e}"

        batch_generator = self.generate_batch(X, self.batch_size, self.num_skips, self.skip_window)
        if sample_weight is None:
            sample_weight = np.ones(len(X))
        for batch_epoch in range(int(self.epochs*len(X)/self.batch_size)):
            epoch = int(batch_epoch*self.batch_size/len(X))
            batch_indices, labels_indices = next(batch_generator)
            batch, labels = sentenceindex_2_word_list(X, batch_indices), sentenceindex_2_word_list(X, labels_indices)
            batch_sample_weight = sample_weight[labels_indices[:,0]]

            feed_dict={
                self.model.X: batch,
                self.model.y: labels.reshape(-1,1),
                self.model.sample_weight: batch_sample_weight
            }

            loss, _ = self.sess.run(
                (self.model.loss, self.model.train_step),
                feed_dict
            )

            if self.log_epochs and not (isinstance(self.log_epochs, float) and epoch%int(1/self.log_epochs)):
                logger.info(epoch_log_str.format(epoch, loss.sum()))

        logger.info("finished fitting :::: loss: " + str(loss.sum()))
        self.final_loss = loss.sum()

        return self

    def transform(self, words):
        '''
        Look up embedding vectors using indices
        words: list of words
        '''
        # make sure all word index are in range
        indices = [self.dictionary[w] for w in words]
        return self.final_embeddings[indices]

    def sort(self, word):
        '''
        Use an input word to sort words using cosine distance in ascending order
        '''
        i = self.dictionary[word]
        vec = self.final_embeddings[i].reshape(1, -1)
        # Calculate pairwise cosine distance and flatten to 1-d
        pdist = sklearn.metrics.pairwise_distances(self.final_embeddings, vec, metric='cosine').ravel()
        return [self.reverse_dictionary[j] for j in pdist.argsort()]


#!/usr/bin/env python
"""For testing Wine dataset"""

# import matplotlib.pyplot as plt
from sklearn.datasets import load_wine
from sklearn.svm import SVR
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn import metrics
from sklearn.preprocessing import StandardScaler, MaxAbsScaler
import matplotlib.pyplot as plt
import itertools
import pandas as pd
import numpy as np
import time

import warnings
warnings.filterwarnings("ignore")

def main():
    """Processes the data and calls test functions"""
    data = dict()
    test = dict()
    scaler = MaxAbsScaler()

    test = {'training' : None, 'testing' : None, 'param_grid' : None, 'scorer' : None}

    param_grid = {'C': 0, # Later this gets set based on data.
                  'epsilon': np.linspace(0, 2, 32),
                  'gamma': np.logspace(-10, 8, 32, base=2)}

    # Process red wine data
    red_data = pd.read_csv("winequality-red.csv")
    samples, target = process_wine_data(red_data, scaler=scaler)
    param_grid['C'] = [best_C_parameter(target)]
    test = {**test, **samples}
    test['param_grid'] = param_grid
    test['scorer'] = metrics.make_scorer(my_score, tolerance=1.5)
    data["red-max-abs-scaled"] = test
    samples, _ = process_wine_data(red_data)
    test = {**test, **samples}
    data["red-unscaled"] = test

    # Process white wine data
    white_data = pd.read_csv("winequality-white.csv", sep=';')
    samples, target = process_wine_data(white_data, scaler=scaler)
    param_grid['C'] = [best_C_parameter(target)]
    test = {**test, **samples}
    test['param_grid'] = param_grid
    data["white-max-abs-scaled"] = test
    samples, _ = process_wine_data(white_data)
    test = {**test, **samples}
    data["white-unscaled"] = test

    # Combine and process both wine data
    combined_data = pd.concat([red_data, white_data])
    combined_data = shuffle(combined_data)
    samples, target = process_wine_data(combined_data, scaler=scaler)
    test = {**test, **samples}
    param_grid['C'] = [best_C_parameter(target)]
    test['param_grid'] = param_grid
    data["combined-max-abs-scaled"] = test
    samples, _ = process_wine_data(combined_data)
    test = {**test, **samples}
    data["combined-unscaled"] = test

    # showData(data)
    data = parameter_test(data)
    quality_prediction(data)
    # showData(data)

def parameter_test(data):
    """Calls best_SVR_parameters() on each data."""
    for key in data:
        print(key)
        data[key]['bestparams'] = best_SVR_parameters(data[key])

    return data

def best_SVR_parameters(data):
    """Returns best parameters for SVR."""

    estimator = SVR(max_iter=50000, kernel='rbf')
    grid = GridSearchCV(estimator, param_grid=data['param_grid'], scoring=data['scorer'], n_jobs=4)
    start = time.time()
    grid.fit(data['training']['x'],
             data['training']['y'],
             sample_weight=get_sample_weights(data['training']['y']))
    print("Time: " + str(time.time() - start))
    print("Best Parameters: " + str(grid.best_params_))
    print("Score: " + str(grid.best_score_))
    return grid.best_params_

def quality_prediction(data):
    """Outputs the mean absolute error and the confusion matrix"""
    for key in data:
        train = data[key]['training']
        best_params = data[key]['bestparams']
        estimator = SVR(**best_params, kernel='rbf')
        estimator.fit(train['x'], train['y'], sample_weight=get_sample_weights(train['y']))

        test = data[key]['testing']
        prediction = estimator.predict(test['x'])

        mae = metrics.mean_absolute_error(test['y'], prediction)
        tolerance = 1
        t_pred = tolerance_prediction(test['y'], prediction, tolerance)
        conf = metrics.confusion_matrix(test['y'], t_pred, labels=range(3,10))

        accuracy = str(my_score(test['y'], prediction, tolerance=1))
        mae_str = str(mae)
        print(key)
        print('mean absolute error: ' + mae_str)
        print('score: ' + accuracy)
        plt.clf()
        plot_confusion_matrix(conf, range(3,10), normalize=False,
                              title=key + '\nmean absolute error = ' + mae_str)
        plt.savefig(key + "-cm", dpi=150)
        plt.clf()
        plot_confusion_matrix(conf, range(3,10), normalize=True,
                              title=key + '\nmean absolute error = ' + mae_str)
        plt.savefig(key + "-cmnorm", dpi=150)
        plt.clf()

def process_wine_data(data, testsize=0.30, scaler=None):
    """Returns features and target of wine data."""
    features = data.loc[:, "fixed acidity":"alcohol"].as_matrix()
    target = data.loc[:, "quality"].as_matrix()
    if scaler:
        scaler.fit(features)
        features = scaler.transform(features) # Scale features to be similar
    x_train, x_test, y_train, y_test = train_test_split(features, target,
                                                        test_size=testsize)
    samples = {'training' : {'x' : x_train, 'y' : y_train},
               'testing' : {'x' : x_test, 'y' : y_test} }
    return samples, target

def best_C_parameter(target):
    mean = np.mean(target)
    std = np.std(target)

    value1 = np.abs(mean + (3 * std))
    value2 = np.abs(mean - (3 * std))

    return max(value1, value2)

def tolerance_prediction(y_values, predictions, tolerance = 0.5):
    result = list()
    for val, pred in zip(y_values, predictions):
        if np.abs(val - pred) < tolerance:
            result.append(val)
        else:
            result.append(np.around(pred))

    return result

def my_score(y_values, predictions, tolerance = 1):
    result = list()
    t_predictions = tolerance_prediction(y_values, predictions, tolerance)
    conf = metrics.confusion_matrix(y_values, t_predictions)
    for row, _i in zip(conf, range(conf.shape[0])):
        for _j in range(conf.shape[0]):
            result.append(np.nan_to_num(row[_i] / np.sum(row)))

    return np.mean(result)

### Taken from scikit-learn.org confusion matrix example
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = np.nan_to_num(cm.astype('float') / cm.sum(axis=1)[:, np.newaxis])
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

def showData(data, indent=0):
    """Displays structure of data."""
    print()
    if isinstance(data, dict):
        for key in data:
            for _i in range(indent):
                print("\t", end="")
            print(key + " ", end="")
            showData(data[key], indent=indent + 1)

def get_sample_weights(target):
    weight = np.absolute(target - 6) + 1
    # return np.power(weight * 4, 2)
    return None

if __name__ == '__main__':
    main()

    # yval = [3,4,5,6,7,8,9,3,3,3,3,3,3,3]
    # ypred = [3,4,5,6,7,8,9,3,4,5,6,7,8,9]

    # print(accuracy_score(yval, ypred, 1.1))