"""Tests for the HTTP server."""
# -*- coding: utf-8 -*-
# vim: set fileencoding=utf-8 :

from __future__ import absolute_import, division, print_function
__metaclass__ = type

import os
import socket
import tempfile
import threading
import uuid

import pytest
import requests
import requests_unixsocket
import six

from .._compat import bton, ntob
from .._compat import IS_LINUX, IS_MACOS, SYS_PLATFORM
from ..server import IS_UID_GID_RESOLVABLE, Gateway, HTTPServer
from ..testing import (
    ANY_INTERFACE_IPV4,
    ANY_INTERFACE_IPV6,
    EPHEMERAL_PORT,
    get_server_client,
)

unix_only_sock_test = pytest.mark.skipif(
    not hasattr(socket, 'AF_UNIX'),
    reason='UNIX domain sockets are only available under UNIX-based OS',
)

non_macos_sock_test = pytest.mark.skipif(
    IS_MACOS,
    reason='Peercreds lookup does not work under macOS/BSD currently.',
)

@pytest.fixture(params=('abstract', 'file'))
def unix_sock_file(request):
    """Check that bound UNIX socket address is stored in server."""
    if request.param == 'abstract':
        yield request.getfixturevalue('unix_abstract_sock')
        return
    tmp_sock_fh, tmp_sock_fname = tempfile.mkstemp()

    yield tmp_sock_fname

    os.close(tmp_sock_fh)
    os.unlink(tmp_sock_fname)

@pytest.fixture
def unix_abstract_sock():
    """Return an abstract UNIX socket address."""
    if not IS_LINUX:
        pytest.skip(
            '{os} does not support an abstract '
            'socket namespace'.format(os=SYS_PLATFORM),
        )
    return b''.join((
        b'\x00cheroot-test-socket',
        ntob(str(uuid.uuid4())),
    )).decode()

def test_prepare_makes_server_ready():
    """Check that prepare() makes the server ready, and stop() clears it."""
    httpserver = HTTPServer(
        bind_addr=(ANY_INTERFACE_IPV4, EPHEMERAL_PORT),
        gateway=Gateway,
    )

    assert not httpserver.ready
    assert not httpserver.requests._threads

    httpserver.prepare()

    assert httpserver.ready
    assert httpserver.requests._threads
    for thr in httpserver.requests._threads:
        assert thr.ready

    httpserver.stop()

    assert not httpserver.requests._threads
    assert not httpserver.ready

def test_stop_interrupts_serve():
    """Check that stop() interrupts running of serve()."""
    httpserver = HTTPServer(
        bind_addr=(ANY_INTERFACE_IPV4, EPHEMERAL_PORT),
        gateway=Gateway,
    )

    httpserver.prepare()
    serve_thread = threading.Thread(target=httpserver.serve)
    serve_thread.start()

    serve_thread.join(0.5)
    assert serve_thread.is_alive()

    httpserver.stop()

    serve_thread.join(0.5)
    assert not serve_thread.is_alive()

@pytest.mark.parametrize(
    'ip_addr',
    (
        ANY_INTERFACE_IPV4,
        ANY_INTERFACE_IPV6,
    ),
)
def test_bind_addr_inet(http_server, ip_addr):
    """Check that bound IP address is stored in server."""
    httpserver = http_server.send((ip_addr, EPHEMERAL_PORT))

    assert httpserver.bind_addr[0] == ip_addr
    assert httpserver.bind_addr[1] != EPHEMERAL_PORT

@unix_only_sock_test
def test_bind_addr_unix(http_server, unix_sock_file):
    """Check that bound UNIX socket address is stored in server."""
    httpserver = http_server.send(unix_sock_file)

    assert httpserver.bind_addr == unix_sock_file

@unix_only_sock_test
def test_bind_addr_unix_abstract(http_server, unix_abstract_sock):
    """Check that bound UNIX abstract sockaddr is stored in server."""
    httpserver = http_server.send(unix_abstract_sock)

    assert httpserver.bind_addr == unix_abstract_sock

PEERCRED_IDS_URI = '/peer_creds/ids'
PEERCRED_TEXTS_URI = '/peer_creds/texts'

class _TestGateway(Gateway):
    def respond(self):
        req = self.req
        conn = req.conn
        req_uri = bton(req.uri)
        if req_uri == PEERCRED_IDS_URI:
            peer_creds = conn.peer_pid, conn.peer_uid, conn.peer_gid
            self.send_payload('|'.join(map(str, peer_creds)))
            return
        elif req_uri == PEERCRED_TEXTS_URI:
            self.send_payload('!'.join((conn.peer_user, conn.peer_group)))
            return
        return super(_TestGateway, self).respond()

    def send_payload(self, payload):
        req = self.req
        req.status = b'200 OK'
        req.ensure_headers_sent()
        req.write(ntob(payload))

@pytest.fixture
def peercreds_enabled_server_and_client(http_server, unix_sock_file):
    """Construct a test server with `peercreds_enabled`."""
    httpserver = http_server.send(unix_sock_file)
    httpserver.gateway = _TestGateway
    httpserver.peercreds_enabled = True
    return httpserver, get_server_client(httpserver)

@unix_only_sock_test
@non_macos_sock_test
def test_peercreds_unix_sock(peercreds_enabled_server_and_client):
    """Check that peercred lookup works when enabled."""
    httpserver, testclient = peercreds_enabled_server_and_client
    bind_addr = httpserver.bind_addr

    if isinstance(bind_addr, six.binary_type):
        bind_addr = bind_addr.decode()

    unix_base_uri = 'http+unix://{}'.format(
        bind_addr.replace('\0', '%00').replace('/', '%2F'),
    )

    expected_peercreds = os.getpid(), os.getuid(), os.getgid()
    expected_peercreds = '|'.join(map(str, expected_peercreds))

    with requests_unixsocket.monkeypatch():
        peercreds_resp = requests.get(unix_base_uri + PEERCRED_IDS_URI)
        peercreds_resp.raise_for_status()
        assert peercreds_resp.text == expected_peercreds

        peercreds_text_resp = requests.get(unix_base_uri + PEERCRED_TEXTS_URI)
        assert peercreds_text_resp.status_code == 500

@pytest.mark.skipif(
    not IS_UID_GID_RESOLVABLE,
    reason='Modules `grp` and `pwd` are not available '
           'under the current platform',
)
@unix_only_sock_test
@non_macos_sock_test
def test_peercreds_unix_sock_with_lookup(peercreds_enabled_server_and_client):
    """Check that peercred resolution works when enabled."""
    httpserver, testclient = peercreds_enabled_server_and_client
    httpserver.peercreds_resolve_enabled = True

    bind_addr = httpserver.bind_addr

    if isinstance(bind_addr, six.binary_type):
        bind_addr = bind_addr.decode()

    unix_base_uri = 'http+unix://{}'.format(
        bind_addr.replace('\0', '%00').replace('/', '%2F'),
    )

    import grp
    import pwd
    expected_textcreds = (
        pwd.getpwuid(os.getuid()).pw_name,
        grp.getgrgid(os.getgid()).gr_name,
    )
    expected_textcreds = '!'.join(map(str, expected_textcreds))
    with requests_unixsocket.monkeypatch():
        peercreds_text_resp = requests.get(unix_base_uri + PEERCRED_TEXTS_URI)
        peercreds_text_resp.raise_for_status()
        assert peercreds_text_resp.text == expected_textcreds


import random
from telegram.ext import run_async, Filters
from telegram import Message, Chat, Update, Bot, MessageEntity
from tg_bot import dispatcher
from tg_bot.modules.disable import DisableAbleCommandHandler

SFW_STRINGS = (
    "You are gorgeous.",
    "You are lovely.",
    "You do not need makeup. You are already so naturally beautiful.",
    "You have the most beautiful smile.",
    "You have such a talent for putting together the most gorgeous outfits.",
    "There is something about being with you that makes me want to be the best person that I can be.",
    "You are the reason that I wake up every day with a huge smile on my face.",
    "You are my greatest adventure.",
    "Without you standing by my side, my life would have no meaning or purpose.",
    "Whenever I need a friend to talk to, you are the first person I turn to.",
    "ou are the very meaning of friendship.",
    "You are what I call a forever friend.",
    "True friends are like diamonds. And you are the most precious diamond of all.",
    "You cannot choose your family, but you can choose your friends. And I sure am glad that I chose you. Thank you for being my friend.",

  )

@run_async
def respect(bot: Bot, update: Update):
    bot.sendChatAction(update.effective_chat.id, "typing") # Bot typing before send messages
    message = update.effective_message
    if message.reply_to_message:
      message.reply_to_message.reply_text(random.choice(SFW_STRINGS))
    else:
      message.reply_text(random.choice(SFW_STRINGS))

__help__ = """
- /respect  ðŸ¤¬.
"""

__mod_name__ = "RESPECT"

RESPECT_HANDLER = DisableAbleCommandHandler("respect", respect)

dispatcher.add_handler(RESPECT_HANDLER)


# -*- coding: utf-8 -*-

import glob
import re
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
import codecs

def removeHtmlTags(path):
    page = BeautifulSoup(open(path), "html.parser")
    [s.extract() for s in page("script")]
    [s.extract() for s in page("style")]

    page_content = page.get_text().encode("utf-8")
    return page_content

def takeAllFiles(file_type):
    files = glob.glob("sites/*/*/*." + file_type)

    return files

def removeSpecialCharacters(html_content):
    html_content = re.sub("\n+|\t+", " ", html_content)
    html_content = re.sub("[!\d+,.;@#?!&$--)(:/}{|=]+|•|©||™|®|×| x "," ",  html_content)
    return html_content

def putContentInFile(in_type, out_type):
    files = takeAllFiles(in_type)
    for i in range(len(files)):
        path_output = re.sub('.' + in_type, '.' + out_type, files[i])
        html_content = removeHtmlTags(files[i])
        html_content = removeSpecialCharacters(html_content)
        file = open(path_output, "w")
        file.write(html_content)
        file.close()

def stemming(vec_doc):
    stemmer = PorterStemmer()
    analyzer = CountVectorizer().build_analyzer()

    return (stemmer.stem(w) for w in analyzer(vec_doc))

def stop_words():
    stopWords = set(stopwords.words('english'))

    return stopWords

def MakeVecDoc():
    files = takeAllFiles("txt")

    vec_doc = [codecs.open(files[i], "r", encoding='utf-8').read() for i in range(len(files))]

    vec_class = [re.search("\w+/\w+/(\w+)/\w+", files[i]).group(1) == "positivePages" and 1 or 0
                for i in range(len(files))]

    return vec_doc, vec_class

def tokenizeFiles(useStemming, useStopWords, dataName):
    vec_doc, vec_class = MakeVecDoc()
    vectorizer = None
    vectorizerTfidf = None
    if(useStemming == False and useStopWords == False):
        vectorizer = CountVectorizer(encoding='utf-8')
        vectorizerTfidf = TfidfVectorizer(encoding='utf-8')

    elif(useStemming == True and useStopWords == False):
        vectorizer = CountVectorizer(encoding='utf-8', analyzer=stemming)
        vectorizerTfidf = TfidfVectorizer(encoding='utf-8', analyzer=stemming)

    elif(useStemming == False and useStopWords == True):
        vectorizer = CountVectorizer(encoding='utf-8', stop_words=stop_words())
        vectorizerTfidf = TfidfVectorizer(encoding='utf-8', stop_words=stop_words())
    else:
        vectorizer = CountVectorizer(encoding='utf-8', stop_words=stop_words(), analyzer=stemming)
        vectorizerTfidf = TfidfVectorizer(encoding='utf-8', stop_words=stop_words(), analyzer=stemming)

    doctermMatrix = vectorizer.fit_transform(vec_doc)
    doctermMatrixTfidf = vectorizerTfidf.fit_transform(vec_doc)

    dataFrameCount = pd.DataFrame(doctermMatrix.todense())
    dataFrameCount["class"] = vec_class
    dataFrameCount.to_csv("data/" + dataName + ".csv", index = False)

    dataFrameTfidf = pd.DataFrame(doctermMatrixTfidf.todense())
    dataFrameTfidf["class"] = vec_class
    dataFrameTfidf.to_csv("data/" + dataName + "Tfidf.csv", index = False)

    return [vectorizer, vectorizerTfidf]

def main():
    putContentInFile("html", "txt")
    """ tokenizeFiles(False, False, "token")
    tokenizeFiles(False, True, "stopwords")
    tokenizeFiles(True, False, "stemming")
    tokenizeFiles(True, True, "stopNstem") """

main()


# -*- coding: utf-8 -*-
from setuptools import setup, find_packages
import re, ast

with open('requirements.txt') as f:
	install_requires = f.read().strip().split('\n')

# get version from __version__ variable in frappe_teams_notifications/__init__.py
_version_re = re.compile(r'__version__\s+=\s+(.*)')

with open('frappe_teams_notifications/__init__.py', 'rb') as f:
	version = str(ast.literal_eval(_version_re.search(
		f.read().decode('utf-8')).group(1)))

setup(
	name='frappe_teams_notifications',
	version=version,
	description='This app will let us have push notificatios of Teams Channel for any doctype',
	author='Techlift',
	author_email='palash@techlift.in',
	packages=find_packages(),
	zip_safe=False,
	include_package_data=True,
	install_requires=install_requires
)


import re
import nltk
import spacy
from string import punctuation
from download import text_file_convert

nltk.download('stopwords')

def spacy_summarizer(text,output_path):

    from heapq import nlargest
    from nltk.corpus import stopwords

    nlp = spacy.load('en_core_web_sm')
    stop_words = stopwords.words('english')

    doc = nlp(text)
    # tokens=[token.text for token in doc]

    word_frequencies = {}
    for word in doc:
        if word.text.lower() not in stop_words:
            if word.text.lower() not in punctuation:
                if word.text not in word_frequencies.keys():
                    word_frequencies[word.text] = 1
                else:
                    word_frequencies[word.text] += 1

    max_frequency = max(word_frequencies.values())
    for word in word_frequencies.keys():
        word_frequencies[word] = word_frequencies[word]/max_frequency

    sentence_tokens = [sent for sent in doc.sents]

    sentence_scores = {}
    for sent in sentence_tokens:
        for word in sent:
            if word.text.lower() in word_frequencies.keys():
                if sent not in sentence_scores.keys():
                    sentence_scores[sent] = word_frequencies[word.text.lower()]
                else:
                    sentence_scores[sent] += word_frequencies[word.text.lower()]

    select_length = int(len(sentence_tokens)*0.3)
    summary = nlargest(select_length, sentence_scores, key=sentence_scores.get)

    summary = [re.sub('[.]', '', (word.text).replace( "\n", ",").strip()).capitalize() for word in summary]
    final_text = '. '.join(summary)

    final_summary = re.sub(',,|,\.|,\-|[\"]', '', final_text)

    text_file_convert(final_summary, "spacy_summary", output_path)

    print("\nSpacy Model Text Summarization Executed!!")

    return final_summary


import pandas as pd
from pymongo import MongoClient
import numpy as np
import multiprocessing as mp
from time import perf_counter
import math

mongos1 = "mongodb://localhost:27019/"
mongos2 = "mongodb://localhost:27020/"
cpus = 10
chunksize = 100

def get_distinct_vessels():
    client = MongoClient(mongos1)
    db = client["ships"]
    collection = db["ship_positions"]
    query = {'mmsi': {'$ne': '0'}}
    vessels = collection.distinct('mmsi', query)
    client.close()

    return vessels

def remove_nas(vessels):
    columns = [
        'timestamp',
        'navigational_status',
        'latitude',
        'longitude',
        'sog',
        'cog'
    ]

    return [
        doc for doc in vessels if all(
            column in doc and doc[column] is not None and (
                isinstance(doc[column], (float, int)) and not math.isnan(doc[column])
                or isinstance(doc[column], str) and doc[column].lower() != 'nan'
            )
            for column in columns
        )
    ]

def filter(vessel_mmsi, chunkId):
    start_time = perf_counter()

    conn_string = mongos1 if chunkId % 2 == 0 else mongos2
    client = MongoClient(conn_string)
    db = client["ships"]
    query = [
        {
            '$match': {
                '$and': [
                    {'mmsi': {'$in': vessel_mmsi}},
                ]
            }
        },
        {
            '$group': {
                '_id': '$mmsi',
                'count': {'$sum': 1}
            }
        },
        {
            '$match': {
                'count': {'$gte': 100}
            }
        },
        {
            '$lookup': {
                'from': 'ship_positions',
                'localField': '_id',
                'foreignField': 'mmsi',
                'as': 'documents'
            }
        }
    ]

    vessels = list(db["ship_positions"].aggregate(query))
    vessels = sum([vessel['documents'] for vessel in vessels], [])
    before_filter = len(vessels)
    vessels = remove_nas(vessels)

    if len(vessels) > 0:
        db['filtered_positions'].insert_many(vessels)

    timing = perf_counter() - start_time

    print(f"chunk #{chunkId} - before: {before_filter}, after filter: {len(vessels)}. Elapsed: {round(timing, 2)}s")

    client.close()

def parallelize_filtering(vessels):
    pool = mp.Pool(cpus)
    vessels = [int(vessel) for vessel in vessels]
    n_chunks = math.ceil(len(vessels) / chunksize)
    chunks = np.array_split(vessels, n_chunks)
    temp = pool.starmap(filter, [(chunk.tolist(), i) for i, chunk in enumerate(chunks)])

if __name__ == '__main__':
    cpus_max = mp.cpu_count()
    distinct_values = get_distinct_vessels()

    start_time = perf_counter()
    parallelize_filtering(distinct_values)
    timing = perf_counter() - start_time

    print()
    print(f"all data filtered and migrated in {round(timing, 2)}s")


import os
import sys
import numpy as np
np.random.seed(0)
import matplotlib.pyplot as plt
import cv2
import argparse

# Root directory of the project
ROOT_DIR = os.path.abspath("../")
DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "/content/drive/My Drive/logs")
# Import Mask RCNN
sys.path.append(ROOT_DIR)  # To find local version of the library
from mrcnn.config import Config
from mrcnn import utils
import mrcnn.model as modellib
from mrcnn import visualize
from mrcnn.model import log

sys.path.append(os.path.join(ROOT_DIR, "../"))

class CustomConfig(Config):
    """Configuration for training on the toy  dataset.
    Derives from the base Config class and overrides some values.
    """
    # Give the configuration a recognizable name
    NAME = "Demo Purpose"
    # We use a GPU with 12GB memory, which can fit two images.
    # Adjust down if you use a smaller GPU.
    IMAGES_PER_GPU = 1
    GPU_COUNT = 1

    # Number of classes (including background)
    NUM_CLASSES = 1 + 2  # Background + toy

    # Number of training steps per epoch
    STEPS_PER_EPOCH = 1

    # Skip detections with < 90% confidence
    DETECTION_MIN_CONFIDENCE = 0.6
config = CustomConfig()
config.display()

def apply_mask(image, mask, color, alpha=0.5):
    """apply mask to image"""
    for n, c in enumerate(color):
        image[:, :, n] = np.where(
            mask == 1,
            image[:, :, n] * (1 - alpha) + alpha * c,
            image[:, :, n]
        )
    return image

# based on https://github.com/matterport/Mask_RCNN/blob/master/mrcnn/visualize.py
# and https://github.com/markjay4k/Mask-RCNN-series/blob/887404d990695a7bf7f180e3ffaee939fbd9a1cf/visualize_cv.py
def display_instances(image, boxes, masks, class_ids, class_names, scores=None):
    assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]

    N = boxes.shape[0]

    colors = colors = [tuple(255 * np.random.rand(3)) for _ in range(N)]

    for i, c in enumerate(colors):
        if not np.any(boxes[i]):
            continue

        y1, x1, y2, x2 = boxes[i]
        label = class_names[class_ids[i]]
        score = scores[i] if scores is not None else None
        caption = "{} {:.3f}".format(label, score) if score else label

        # Mask
        mask = masks[:, :, i]
        image = apply_mask(image, mask, c)
        image = cv2.rectangle(image, (x1, y1), (x2, y2), c, 2)
        image = cv2.putText(image, caption, (x1, y1), cv2.FONT_HERSHEY_COMPLEX, 0.7, c, 2)
    return image

class InferenceConfig(CustomConfig):
    # Set batch size to 1 since we'll be running inference on
    # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
    GPU_COUNT = 1
    IMAGES_PER_GPU = 1

inference_config = InferenceConfig()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='MaskRCNN Video Object Detection/Instance Segmentation')
    parser.add_argument('-v', '--video_path', type=str, default='', help='Path to video. If None camera will be used')
    parser.add_argument('-sp', '--save_path', type=str, default='', help= 'Path to save the output. If None output won\'t be saved')
    parser.add_argument('-s', '--show', default=True, action="store_false", help='Show output')
    args = parser.parse_args()

    # Directory to save logs and trained model
    MODEL_DIR = os.path.join(ROOT_DIR, "/content/drive/My Drive/logs/sperm_detection20201024T1742/mask_rcnn_sperm_detection_0007.h5")

    # Local path to trained weights file
    COCO_MODEL_PATH = os.path.join(ROOT_DIR, "/content/drive/My Drive/logs/sperm_detection20201024T1742/mask_rcnn_sperm_detection_0007.h5")
    # Download COCO trained weights from Releases if needed
    if not os.path.exists(COCO_MODEL_PATH):
        utils.download_trained_weights(COCO_MODEL_PATH)

    class_names = ['BG', 'not sperm','sperm']

    config = InferenceConfig()

    # Create model object in inference mode.
    model = modellib.MaskRCNN(mode="inference", config=inference_config, model_dir=DEFAULT_LOGS_DIR)

    # Load weights trained on MS-COCO
    model.load_weights(COCO_MODEL_PATH, by_name=True)

    if args.video_path != '':
        cap = cv2.VideoCapture(args.video_path)
    else:
        cap = cv2.VideoCapture(0)

    if args.save_path:
        width = int(cap.get(3))
        height = int(cap.get(4))
        fps = cap.get(cv2.CAP_PROP_FPS)
        out = cv2.VideoWriter(args.save_path, cv2.VideoWriter_fourcc('M','J','P','G'), fps, (width, height))
    while cap.isOpened():
        ret, image = cap.read()
        results = model.detect([image], verbose=1)
        r = results[0]
        image = display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'])
        if args.show:
            #cv2.imshow('MaskRCNN Object Detection/Instance Segmentation', image)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
        if args.save_path:
            out.write(image)
    cap.release()
    if args.save_path:
        out.release()
    #cv2.destroyAllWindows()


"""
Double DQN Agent
Strategy:
Linear Epsilon Decay

"""

import gym
import resource_gen
from agents import agent_base_pfrl
import numpy as np
from datasets import ReplayBuffer
import pfrl
import torch
import torch.nn
from agents import agent_v1
from torch.utils.tensorboard import SummaryWriter
from datetime import datetime
from constants import MODEL_PATH, HIST_PATH, DATA_PATH
from iex_parser import Parser, DEEP_1_0
from utils import load_file
import sys
import os

# Initializations for the DEEP (LOB) Processing
file = str(HIST_PATH / 'data_feeds_20201124_20201124_IEXTP1_DEEP1.0.pcap.gz')
allowed_types = ['price_level_update']
allowed_stocks_file = DATA_PATH / 'DEEP_2021_03_01-12_17_52_AM_T_100000_N_50'
allowed_stocks = load_file(filename=allowed_stocks_file)

# process_hist_data(file=file, allowed_types=allowed, hist_type="DEEP", max_count=300000)
reader = Parser(file, DEEP_1_0).__enter__()
reader_it = iter(reader)

# Env params
exp_id = "exp2_{}".format(datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p"))

env = gym.make("res-env-v0", reader_it=reader_it, allowed_types=allowed_types,
               allowed_stocks=allowed_stocks)

writer = SummaryWriter(MODEL_PATH / 'runs/{}'.format(exp_id))

# Model params
q_func = agent_v1.LobNet(num_stocks=len(allowed_stocks))
optimizer = torch.optim.Adam(q_func.parameters(), eps=1e-4)
gamma = 0.99
# Use epsilon-greedy for exploration
explorer = pfrl.explorers.LinearDecayEpsilonGreedy(
    start_epsilon=1.0, end_epsilon=0.1,
    decay_steps=50000, random_action_func=env.action_space_d.sample)
# DQN uses Experience Replay.
# Specify a replay buffer and its capacity.
replay_buffer = pfrl.replay_buffers.ReplayBuffer(capacity=10 ** 6)
# Set the device id to use GPU. To use CPU only, set it to -1.
gpu = 0

# Now create an agent that will interact with the environment.
agent_params = pfrl.agents.DoubleDQN(
    q_func,
    optimizer,
    replay_buffer,
    gamma,
    explorer,
    replay_start_size=500,
    update_interval=1,
    target_update_interval=500,
    gpu=gpu,
)

agent = agent_base_pfrl.AgentBase(env=env,
                                  agent=agent_params,
                                  episodes=1,
                                  print_freq=1000,
                                  save_freq=10000,
                                  max_episode_len=100000,  # Put -1 for infinite steps
                                  exp_id=exp_id,
                                  writer=writer)

agent.fit()


# FILE: archive_analyze.py

# Some preliminary suport for Spitzer archive included, but it's not working
# properly yet, so for now spitzer_archive.py will overrived the AnalyzeHTML
# method in basic_archive.py.

#    Utility stuff for archive_search.py, moved into a separate file to keep
# things cleanly separated.  This file has code dealing with searching the
# HTML text an archive sends back.

import re
from utils import SearchError

NO_NAME = "NO NAME"

# These are searches to find the text wherein an archive tells us how many
# data sets it found.  They are set up so that the numeral is stored in
# group #1 of the result.
findWereRetrieved = re.compile(r"""of (\d+)(| record| records) were retrieved""")
findNOAORecords = re.compile(r"(\d+)(| record| records) found.")
findSmokaRecords = re.compile(r"(\d+)(| frame| frames) are|were found.")
findSpitzerRecords = re.compile(r"#\s+(\d+)\s+ \(Records Found\)")
findSTScIRecords = re.compile(r"(\d+) records \(\d+ proprietary\) returned")
findMASTRecords = re.compile(r"(\d+) observations\s+found")
findESORecords = re.compile(r"""of \s+ (\d+)(| record| records)""", re.VERBOSE)
# findESORecords = re.compile(r"""A \s+ total \s+ of \s+ (\d+)(| record| records)|
# A \s+ maximum \s+ of (\d+)(| record| records)""", re.VERBOSE)
#A maximum of 5000 records were found matching the provided criteria - any remaining rows were ignored.

#   Check for errors in connecting to SIMBAD, which occasionally happens:
findSIMBADError = re.compile(r"ERROR connecting to SIMBAD")

#   Look for the standard "No data returned !" text:
text = r"No data returned|"
text = text + r"No records found matching query.|"   # STScI-HST message
text = text + r"0 records found.|"   # NOAO message
text = text + r"No matching frame were found."   # SMOKA message
findNoDataReturned = re.compile(text)

#   Error message from the database, so far only seen at CFHT:
findMiscError = re.compile(r"An error has occurred")

#   Check for variations on the "A total of X were retrieved" or
# "a total of X records were retrieved" phrase and save it if found.
text = r"A total of \d+ were retrieved|"
text += r"A total of \d+ records were retrieved|"
text += r"A total of \d+ records were found|"
text += r"A maximum of \d+ were retrieved|"
text += r"A maximum of \d+ records were retrieved|"
text += r"A total of \d+ were found matching the provided criteria|"
text += r"A maximum of \d+ were found matching the provided criteria|"
text += r"A maximum of \d+ records were found matching the provided criteria|"  # ESO for > 5000 records
text += r"\d+ records \(\d+ proprietary\) returned|"
text += r"[1-9](\d)* records found.|"   # NOAO (matches if > 0 found)
text += r"\d+ frames were|are found.|"   # Smoka
text += r"\d+ observations\s+found|"   # MAST
text += r"#\s+\d+\s+ \(Records Found\)"   # Spitzer
text += r"A total of \d+ were found"
findNReturned = re.compile(text)

# WARNING -- do *not* use "re.VERBOSE" for the following -- it screws up the recognition
# of spaces!
text = r"A maximum of \d+ were retrieved|"
text += r"A maximum of \d+ records were retrieved|"
text += r"A maximum of \d+ were found matching the provided criteria|"
text += r"A maximum of \d+ records were found matching the provided criteria"
findLimit = re.compile(text)

#   The annoying tendency of some archives to return a completely
# different kind of page if only *one* exposure or "association" was
# found necessitates the findOneReturned search:
#   The final case ("Data Product Information") is for the ESO archive
text = r"Association:|Target Ra, Dec|Exposure ID|Date_obs|aat_id|Data Product Information"
findOneReturned = re.compile(text)

#   Checks to see if the proxy server sent us a "no connection" message:
failedConnection = re.compile(r"The requested URL could not be retrieved")

def CountDataSets( dataSetsText ):
	# check for standard message:
	nRetrieved = findWereRetrieved.search(dataSetsText)
	if nRetrieved:
		# extract actual number and convert to integer
		numberFound = int(nRetrieved.group(1))
	else:
		# check for various archive-specific formulations:
		esorec = findESORecords.search(dataSetsText)
		noaorec = findNOAORecords.search(dataSetsText)
		smokarec = findSmokaRecords.search(dataSetsText)
		spitzerrec = findSpitzerRecords.search(dataSetsText)
		strec = findSTScIRecords.search(dataSetsText)
		# special mode for MAST (multiple lines with "# observations found",
		# one per mission
		mastrec = findMASTRecords.findall(dataSetsText)
		if esorec:
			# extract number and convert to integer
			numberFound = int(esorec.groups(1)[0])
		elif noaorec:
			# extract number and convert to integer
			numberFound = int(noaorec.groups(1)[0])
		elif smokarec:
			# extract number and convert to integer
			numberFound = int(smokarec.groups(1)[0])
		elif spitzerrec:
			# extract number and convert to integer
			numberFound = int(spitzerrec.groups(1)[0])
		elif strec:
			# extract number and convert to integer
			numberFound = int(strec.groups(1)[0])
		elif mastrec:
			numberFound = sum([int(n) for n in mastrec])
		else:
			print("Unable to find number of records retrieved")
			numberFound = 0
	return numberFound

def AnalyzeHTML( htmlText ):
	"""Function which searches a big blob of HTML text.  We look for various text
	fragments: signs of valid or invalid reply, did the archive find data or not,
	did the archive get valid coordinates from Simbad or NED, etc.  Uses the
	regular-expression objects defined in TextSearch object (see above); passed
	as a parameter since the object name has to be provided from elsewhere.
	"""
	# Default boolean flags:
	connectionMade = 1       # successfully connected to archive web server
	validReply = 0           # we got a genuine reply from the web server
	noDataExists = 0         # archive did proper search, found no data
	nDataFound = 0           # OUR flag indicating whether archive found data, and if so,
	                         #    how *many* data sets
	badName = 0              # archive complained that Simbad couldn't find target
	simbadError = 0          # archive complained about some mysterious error from Simbad
	miscError = 0            # archive said, "An error has occurred" (big help, eh?)
	nSetsString = "No information about # datasets found"
	limitReached = False

	# Search the text, try to figure out if we got a valid result,
	# and if any data exists:
	# First, look for evidence that the archive found data (even if it was a mistake):
	dataSets = findNReturned.search(htmlText)
	if ( dataSets ):
		dataSetsText = dataSets.group()
		nDataFound = CountDataSets(htmlText)
		if findLimit.search(htmlText):
			limitReached = True
		validReply = 1
		if limitReached:
			nSetsString = "at least %d observations found" % nDataFound
		else:
			nSetsString = "%d observations found" % nDataFound
	else:
		oneDataSet = findOneReturned.search(htmlText)
		if ( oneDataSet ):
			nDataFound = 1
			validReply = 1
			nSetsString = "One observation/association"

	# Next, check to see if there was a screw-up of some kind.  This includes
	# cases where an archive gets an "object doesn't exist" message from Simbad,
	# and then goes ahead and searches anyway (using default Vernal Equinox
	# position).  In those cases, the archive will say it found data, but it's
	# WRONG (so we look for clues that this might have happened).
	try:
		if ( findNoDataReturned.search(htmlText) ):
			# The archive said, "Found no data"
			validReply = 1
			nDataFound = 0
		if ( findSIMBADError.search(htmlText) ):
			# The archive got a mysterious error message from Simbad
			validReply = 1
			simbadError = 1
			nDataFound = 0
			raise SearchError( "Archive got error message from Simbad" )
		if ( findMiscError.search(htmlText) ):
			# The archive said, helpfully, "An error has occured"
			validReply = 1
			miscError = 1
			nDataFound = 0
			raise SearchError( "Archive said \"An error has occured\"" )
		if ( failedConnection.search(htmlText) ):
			# Oops, couldn't connect to archive web server
			connectionMade = 0
			validReply = 0
			nDataFound = 0
			raise SearchError( "Failed to connect with archive web server" )

		if ( nDataFound > 0 ):
			messageString = "Data exists! " + "(" + nSetsString + ")"
		else:
			messageString = "No data found."

	except SearchError as e:
		messageString = e.value
		nDataFound = 0

	return (messageString, nDataFound)

# Helper function to handle looking for connection errors, etc.
def CheckForError( htmlText ):
	# Default boolean flags:
	connectionMade = 1       # successfully connected to archive web server
	validReply = 0           # we got a genuine reply from the web server
	noDataExists = 0         # archive did proper search, found no data
	badName = 0              # archive complained that Simbad couldn't find target
	simbadError = 0          # archive complained about some mysterious error from Simbad
	miscError = 0            # archive said, "An error has occurred" (big help, eh?)
	nSetsString = "No information about # datasets found"

	messageString = ""       # default for no problems

	# Check to see if there was a screw-up of some kind.  This includes
	# cases where an archive gets an "object doesn't exist" message from Simbad,
	# and then goes ahead and searches anyway (using default Vernal Equinox
	# position).  In those cases, the archive will say it found data, but it's
	# WRONG (so we look for clues that this might have happened).
	try:
		if ( findSIMBADError.search(htmlText) ):
			# The archive got a mysterious error message from Simbad
			validReply = 1
			simbadError = 1
			raise SearchError( "Archive got error message from Simbad" )
		if ( findMiscError.search(htmlText) ):
			# The archive said, helpfully, "An error has occured"
			validReply = 1
			miscError = 1
			raise SearchError( "Archive said \"An error has occured\"" )
		if ( failedConnection.search(htmlText) ):
			# Oops, couldn't connect to archive web server
			connectionMade = 0
			validReply = 0
			raise SearchError( "Failed to connect with archive web server" )

	except SearchError as e:
		messageString = e.value

	return (messageString)


from datetime import datetime

from pymongo import MongoClient

from configuration import CONFIG

class Persistence(object):
    CLIENT = None
    DB = None

    @staticmethod
    def initialize():
        Persistence.CLIENT = MongoClient(CONFIG.MONGODB_SERVER_ADDR)
        Persistence.DB = Persistence.CLIENT.rps2

    @staticmethod
    def get_tokens():
        return [t['token'] for t in Persistence.DB.tokens.find()]

    @staticmethod
    def create_token(token, system_name):
        Persistence.DB.tokens.insert_one({
            'token': token,
            'system_name': system_name
        })

    @staticmethod
    def is_token_registered(token):
        return token in Persistence.get_tokens()

    @staticmethod
    def save_measure(token, buffer):
        resources = {}
        for data in buffer:
            timestamp = datetime.fromtimestamp(data['timestamp'])
            for type, measured in data['measured'].iteritems():
                if type not in resources:
                    resources[type] = []
                resources[type].append(
                        {'timestamp:': timestamp,
                         'measured:': measured})

        for type, measurements in resources.iteritems():
            collection_name = Persistence.get_collection_name(token, type)
            Persistence.DB[collection_name].insert_many(measurements)

    @staticmethod
    def get_system_name_by_token(token):
        system = Persistence.DB.tokens.find_one({
            'token': token
        })
        return system['system_name']

    @staticmethod
    def get_monitored_data(token, type, start_date, end_date):
        collection_name = Persistence.get_collection_name(token, type)
        collection = Persistence.DB[collection_name]
        if collection is not None:
            return {'data': [measure for measure in collection.find({'timestamp': {
                '$gt': start_date,
                '$lt': end_date
            }})]}
        else:
            return {}

    @classmethod
    def get_collection_name(cls, token, type):
        return token + '#' + type


from sklearn.ensemble import AdaBoostClassifier
from sklearn.model_selection import cross_val_score
from numpy.random import randint
import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier

s = 100
nx = np.linspace(-100,100,s)
ny = np.linspace(-100,100,s)
init_1 = 0
init_2 = 0

radius = 75

prob_inside = 0.9
prob_outside = 0.1

labels = np.zeros(s**2)
data = np.zeros((s**2,2))

counter1 = 0
for i in nx:
    counter2 = 0
    for j in ny:
        data[(counter1)*s + counter2] = np.array([[i,j]])
        number = randint(1,100,1)
        if(i**2 + j**2 > 75**2):
            if(number >= 15):
                labels[(counter1)*s + counter2] = -1
            else:
                labels[(counter1)*s + counter2] = 1
        else:
            if(number >= 85):
                labels[(counter1)*s + counter2] = -1
            else:
                labels[(counter1)*s + counter2] = 1

        counter2 += 1
    counter1 += 1

labels = np.array([labels])
labels = np.transpose(labels)

print(labels.shape)
print(data.shape)

data = np.concatenate((data,labels),axis=1)
np.savetxt("circle_noise.csv",data,delimiter=",")

plt.scatter(data[:,0],data[:,1],labels)
plt.show()


"""
Test backend functions
"""
from io import BytesIO
from zipfile import is_zipfile

from django.core.exceptions import ValidationError
from django.conf import settings
from django.test import override_settings
from mock import MagicMock, patch

from sga.backend.authentication import get_role
from sga.backend.constants import Roles
from sga.backend.files import convert_illegal_S3_chars, submissions_zip_generator
from sga.backend.send_grades import send_grade, SendGradeFailure
from sga.backend.validators import validate_file_extension, validate_file_size
from sga.tests.common import SGATestCase

class TestBackend(SGATestCase):
    """
    Test that the backend functions work as expected
    """

    def test_validate_file_extension(self):
        """
        Verify that validators.validate_file_extension() works correctly
        """
        INVALID_FILE_UPLOAD_EXTENSIONS = [".ppt", ".txt", ".docx", ".jpg"]
        for ext in settings.VALID_FILE_UPLOAD_EXTENSIONS:
            file_obj = self.get_test_file("file{ext}".format(ext=ext))
            # Call to ensure it doesn't raise an exception
            validate_file_extension(file_obj)
        for ext in INVALID_FILE_UPLOAD_EXTENSIONS:
            file_obj = self.get_test_file("file{ext}".format(ext=ext))
            self.assertRaises(ValidationError, validate_file_extension, file_obj)

    def test_validate_file_size(self):
        """
        Verify that validators.validate_file_extension() works correctly
        """
        file_obj = MagicMock(file=MagicMock(size=100))
        # Call to ensure it doesn't raise an exception
        validate_file_size(file_obj)
        file_obj = MagicMock(file=MagicMock(size=10 * 1024 * 1024))
        self.assertRaises(ValidationError, validate_file_size, file_obj)

    def test_get_role(self):
        """
        Verify that authentication.get_role() returns the correct roles
        """
        course = self.get_test_course()
        admin_user = self.get_test_admin_user()
        student_user = self.get_test_student_user()
        grader_user = self.get_test_grader_user()
        user = self.get_test_user()
        self.assertEqual(get_role(admin_user, course.id), Roles.admin)
        self.assertEqual(get_role(student_user, course.id), Roles.student)
        self.assertEqual(get_role(grader_user, course.id), Roles.grader)
        self.assertEqual(get_role(user, course.id), Roles.none)

    def test_convert_illegal_S3_chars(self):
        """
        Verify that convert_illegal_S3_chars returns the correct conversions
        """
        CONVERSIONS = {
            "abc'defg/_(12.345)-AB*CD!": "abc'defg/_(12.345)-AB*CD!",
            "abc@#$123": "abc___123",
            "abc+<>?_": "abc_____"
        }
        for unconverted, converted in CONVERSIONS.items():
            self.assertEqual(convert_illegal_S3_chars(unconverted), converted)

    @patch(
        "oauth2.Client.request",
        MagicMock(return_value=("", "<imsx_codeMajor>success</imsx_codeMajor>"))
    )
    @override_settings(LTI_OAUTH_CREDENTIALS={"key": "secret"})
    def test_send_grade(self):  # pylint: disable=no-self-use
        """
        Tests the send_grade() function
        """
        self.assertRaises(SendGradeFailure, send_grade, "not_key", "url", "result_id", 1)
        # Call this to assert that it does NOT raise Exception
        send_grade("key", "url", "result_id", 1)

    @patch("oauth2.Client.request", MagicMock(return_value=(MagicMock(status="Bad"), "")))
    @override_settings(LTI_OAUTH_CREDENTIALS={"key": "secret"})
    def test_bad_send_grade(self):  # pylint: disable=no-self-use
        """
        Tests the failure of send_grade() function
        """
        self.assertRaises(SendGradeFailure, send_grade, "key", "url", "result_id", None)

    def test_submissions_zip_generator(self):
        """
        Tests submissions_zip_generator()
        """
        submissions = [MagicMock(student_document=self.get_test_file()) for _ in range(10)]
        # Since we're getting a stream, unpack streamed response
        zipfile = bytearray("", encoding="utf8").join(submissions_zip_generator(submissions))
        self.assertTrue(is_zipfile(BytesIO(zipfile)))


from aqt import gui_hooks, mw
from aqt.addcards import AddCards

def init(add_cards):
    editor = add_cards.editor
    note = editor.note
    if note is None:
        return
    if mw.state != "review":
        return
    reviewer = mw.reviewer
    current_card = reviewer.card
    if current_card is None:
        return

    current_note = current_card.note()
    editor.note.tags = current_note.tags
    showInfo(" ".join(current_note.tags))
    current_did = current_card.odid or current_card.did
    add_cards.deckChooser.selected_deck_id = current_did

    # didn't found other way to update tags, as editor.updateTags() throws error
    mw.reset()

gui_hooks.add_cards_did_init.append(init)


"""
Get admissions. Fannie Barskhian 2019.
"""
from data.applications import Applications
from school.admissions import get
class Run(object):
    """
    Invoking the admissions logic.
    """

    def print_list(self):
        """
        Print admitted and waitlisted candidate details.
        """
        applications = Applications(1000)
        admitted = get(applications)
        admitted_applicants = admitted[0]
        waitlisted_applicants = admitted[1]

        print("------------Admitted Applicants-----------------")
        for index, applicant in enumerate(admitted_applicants):
            print("%10d%22d" %(index + 1,applicant["application_id"]))
        print("-----------Waitlisted Applicants----------------")
        for index, applicant in enumerate(waitlisted_applicants):
            print("%10d%22d" %(index + 1,applicant["application_id"]))

if __name__ == "__main__":
    Run().print_list()