import os
import sys
import datetime
import github
import collections

DEBUG = False

past_days_30 = datetime.datetime.now() + datetime.timedelta(-30)
past_days_60 = datetime.datetime.now() + datetime.timedelta(-60)

def debug(message):
    if DEBUG:
        print("%s %s" % (datetime.datetime.now(), message))

org_members_cache = set()

def is_in_org(api_org, author):

    if len(org_members_cache) == 0:
        for member in api_org.get_members():
            org_members_cache.add(member.login)

    return author.login in org_members_cache

class Contributor(object):
    def __init__(self):
        self.stats = []
        self.user = None

    def extend(self, stats):
        self.stats.extend(stats)

    def _stats_this_month(self):
        return filter(lambda week: week.w >= past_days_30, self.stats)

    def _stats_last_month(self):
        return filter(lambda week: past_days_30 >= week.w >= past_days_60, self.stats)

    @staticmethod
    def _score(stats):
        result = 0
        for stat in stats:
            commits, additions, deletions = stat.c, stat.a, stat.d
            # Somewhat arbitrarily, value commits at 10 "points" a piece. Code is worth 1 point for
            # 10 lines, capped at 100 points for deletions, 100 for additions each week.
            result += 10 * commits + min(100, int(additions/10)) + min(100, int(deletions/10))
        return int(result)

    def score_this_month(self):
        debug("Computing this-month score for %s" % self.user)
        stats = self._stats_this_month()
        return Contributor._score(stats)

    def score_last_month(self):
        debug("Computing last-month score for %s" % self.user)
        stats = self._stats_last_month()
        return Contributor._score(stats)

class ScoreSummary(object):
    def __init__(self, contributors, scores):
        self.contributors = contributors
        self.scores = scores

    def __str__(self):
        result = []

        for contributor_id in self.scores.keys():
            score_this_month, score_last_month = self.scores[contributor_id]
            result.append((self.contributors[contributor_id].user.name,
                           self.contributors[contributor_id].user.login,
                           self.contributors[contributor_id].user.avatar_url,
                           score_this_month, score_last_month))

        return '\n'.join([','.join(str(y) for y in x) for x in sorted(result, reverse=True,
                                                                      key=lambda t: t[3])])

def main(github_organization, github_token):
    print("Generating your report.")
    print("This might take a while depending on the number of repos in your organization.")
    print("Please wait...")

    api_client = github.Github(github_token)
    api_org = api_client.get_organization(github_organization)
    repos = api_org.get_repos()

    contributors = collections.defaultdict(Contributor)

    for repo in repos:
        debug("Inspecting repository %s" % repo)

        # Only inspect private repos and repos that have been pushed to in the last 60 days.
        if repo.private and repo.pushed_at > past_days_60:
            stats_contributors = repo.get_stats_contributors()

            if stats_contributors is not None:
                for stats_contributor in stats_contributors:
                    debug("Logging stats for contributor %s" % stats_contributor.author)

                    if not is_in_org(api_org, stats_contributor.author):
                        continue

                    contributors[stats_contributor.author.id].user = stats_contributor.author
                    contributors[stats_contributor.author.id].stats.extend(stats_contributor.weeks)

    scores = {}
    for contributor in sorted(contributors.keys()):
        this_month = contributors[contributor].score_this_month()
        last_month = contributors[contributor].score_last_month()

        if this_month != 0 or last_month != 0:
            scores[contributor] = (this_month, last_month)

    score_summary = str(ScoreSummary(contributors, scores))
    print()
    print("User Name,User ID,User Avatar URL,Score This Month,Score Last Month")
    print(score_summary)

def usage():
    print("Usage: python3 %s <organization_name>" % sys.argv[0])
    sys.exit(1)

if __name__ == '__main__':
    if 'GITHUB_TOKEN' not in os.environ:
        usage()
    if len(sys.argv) != 2:
        usage()
    main(sys.argv[1], os.environ['GITHUB_TOKEN'])


############################################################
# -*- coding: utf-8 -*-
#
#       #   #  #   #   #    #
#      ##  ##  #  ##  #    #
#     # # # #  # # # #    #  #
#    #  ##  #  ##  ##    ######
#   #   #   #  #   #       #
#
# Python-based Tool for interaction with the 10micron mounts
# GUI with PyQT5 for python
#
# written in python3, (c) 2019-2023 by mworion
# Licence APL2.0
#
###########################################################
# standard libraries
import socket

# external packages
import wakeonlan

# local import
from mountcontrol.mount import checkFormatMAC

class SettMount(object):
    """
    """

    def __init__(self):
        self.ui.mountOn.clicked.connect(self.mountBoot)
        self.ui.mountOff.clicked.connect(self.mountShutdown)
        self.ui.mountHost.editingFinished.connect(self.mountHost)
        self.ui.port3492.clicked.connect(self.mountHost)
        self.ui.port3490.clicked.connect(self.mountHost)
        self.ui.mountMAC.editingFinished.connect(self.mountMAC)
        self.ui.bootRackComp.clicked.connect(self.bootRackComp)
        self.ui.waitTimeMount.valueChanged.connect(self.setWaitTime)
        self.ui.waitTimeMountFlip.valueChanged.connect(self.setWaitTimeFlip)
        self.ui.clockSync.stateChanged.connect(self.toggleClockSync)
        self.ui.copyFromTelescopeDriver.clicked.connect(self.updateTelescopeParametersToGui)
        self.app.mountOn.connect(self.mountBoot)
        self.app.mountOff.connect(self.mountShutdown)
        self.app.mount.signals.settingDone.connect(self.setMountMAC)
        self.app.mount.signals.firmwareDone.connect(self.updateFwGui)
        self.app.update3s.connect(self.updateTelescopeParametersToGuiCyclic)
        self.app.update30s.connect(self.syncClock)

    def initConfig(self):
        """
        :return:
        """
        config = self.app.config['mainW']
        self.ui.mountHost.setText(config.get('mountHost', ''))
        self.ui.port3492.setChecked(config.get('port3492', True))
        self.mountHost()
        self.ui.mountMAC.setText(config.get('mountMAC', ''))
        self.mountMAC()
        self.ui.mountWolAddress.setText(config.get('mountWolAddress',
                                                   '255.255.255.255'))
        self.ui.mountWolPort.setText(config.get('mountWolPort', '9'))
        self.ui.rackCompMAC.setText(config.get('rackCompMAC', ''))
        self.ui.waitTimeMount.setValue(config.get('waitTime', 0))
        self.ui.waitTimeMountFlip.setValue(config.get('waitTimeFlip', 0))
        self.ui.automaticTelescope.setChecked(config.get('automaticTelescope', False))
        self.ui.automaticWOL.setChecked(config.get('automaticWOL', False))
        self.ui.syncTimeNone.setChecked(config.get('syncTimeNone', True))
        self.ui.syncTimeCont.setChecked(config.get('syncTimeCont', False))
        self.ui.syncTimeNotTrack.setChecked(config.get('syncTimeNotTrack', False))
        self.ui.clockSync.setChecked(config.get('clockSync', False))
        self.toggleClockSync()

        if self.ui.automaticWOL.isChecked():
            self.mountBoot()
        return True

    def storeConfig(self):
        """
        :return:
        """
        config = self.app.config['mainW']
        config['mountHost'] = self.ui.mountHost.text()
        config['mountMAC'] = self.ui.mountMAC.text()
        config['mountWolAddress'] = self.ui.mountWolAddress.text()
        config['mountWolPort'] = self.ui.mountWolPort.text()
        config['rackCompMAC'] = self.ui.rackCompMAC.text()
        config['waitTime'] = self.ui.waitTimeMount.value()
        config['waitTimeFlip'] = self.ui.waitTimeMountFlip.value()
        config['port3492'] = self.ui.port3492.isChecked()
        config['automaticTelescope'] = self.ui.automaticTelescope.isChecked()
        config['automaticWOL'] = self.ui.automaticWOL.isChecked()
        config['syncTimeNone'] = self.ui.syncTimeNone.isChecked()
        config['syncTimeCont'] = self.ui.syncTimeCont.isChecked()
        config['syncTimeNotTrack'] = self.ui.syncTimeNotTrack.isChecked()
        config['clockSync'] = self.ui.clockSync.isChecked()
        return True

    def mountBoot(self):
        """
        :return:
        """
        bAddress = self.ui.mountWolAddress.text().strip()
        bPort = self.ui.mountWolPort.text().strip()
        bPort = (int(bPort) if bPort else 0)
        suc = self.app.mount.bootMount(bAddress=bAddress, bPort=bPort)
        if suc:
            self.msg.emit(0, 'Mount', 'Command',
                          'Sent boot command to mount')
        else:
            self.msg.emit(2, 'Mount', 'Command', 'Mount cannot be booted')
        return suc

    def mountShutdown(self):
        """
        :return:
        """
        suc = self.app.mount.shutdown()
        if suc:
            self.msg.emit(0, 'Mount', 'Command', 'Shutting mount down')
        else:
            self.msg.emit(2, 'Mount', 'Command', 'Mount cannot be shutdown')
        return suc

    def bootRackComp(self):
        """
        :return:
        """
        MAC = self.ui.rackCompMAC.text()
        MAC = checkFormatMAC(MAC)
        if MAC is not None:
            wakeonlan.send_magic_packet(MAC)
            self.msg.emit(0, 'Rack', 'Command',
                          'Sent boot command to rack computer')
            return True
        else:
            self.msg.emit(2, 'Rack', 'Command',
                          'Rack computer cannot be booted')
            return False

    def mountHost(self):
        """
        :return: true for test purpose
        """
        if self.ui.port3492.isChecked():
            port = 3492
        else:
            port = 3490

        host = self.ui.mountHost.text()
        if not host:
            return False
        try:
            socket.gethostbyname(host)
        except Exception as e:
            self.msg.emit(2, 'Mount', 'Setting error', f'{e}')
            return False

        self.app.mount.host = (host, port)
        self.app.hostChanged.emit()
        return True

    def mountMAC(self):
        """
        :return: true for test purpose
        """
        self.app.mount.MAC = self.ui.mountMAC.text()
        return True

    def setMountMAC(self, sett=None):
        """
        :param sett:
        :return: true for test purpose
        """
        if sett is None:
            return False
        if sett.addressLanMAC is None:
            return False
        if not sett.addressLanMAC:
            return False

        self.app.mount.MAC = sett.addressLanMAC
        self.ui.mountMAC.setText(self.app.mount.MAC)
        return True

    def setWaitTime(self):
        """
        :return: true for test purpose
        """
        self.app.mount.waitTime = self.ui.waitTimeMount.value()
        return True

    def setWaitTimeFlip(self):
        """
        :return: true for test purpose
        """
        self.app.mount.waitTimeFlip = self.ui.waitTimeMountFlip.value()
        return True

    def updateFwGui(self, fw):
        """
        :return:    True if ok for testing
        """
        self.guiSetText(self.ui.product, 's', fw.product)
        self.guiSetText(self.ui.vString, 's', fw.vString)
        self.guiSetText(self.ui.fwdate, 's', fw.date)
        self.guiSetText(self.ui.fwtime, 's', fw.time)
        self.guiSetText(self.ui.hardware, 's', fw.hardware)
        return True

    def toggleClockSync(self):
        """
        :return:
        """
        enableSync = self.ui.clockSync.isChecked()
        self.ui.syncTimeNone.setEnabled(enableSync)
        self.ui.syncTimeCont.setEnabled(enableSync)
        self.ui.syncTimeNotTrack.setEnabled(enableSync)
        self.ui.clockOffset.setEnabled(enableSync)
        self.ui.clockOffsetMS.setEnabled(enableSync)
        if enableSync:
            self.app.mount.startClockTimer()
        else:
            self.app.mount.stopClockTimer()
        return True

    def syncClock(self):
        """
        :return:
        """
        noSync = self.ui.syncTimeNone.isChecked()
        if noSync:
            return False
        if not self.deviceStat['mount']:
            return False

        doSyncNotTrack = self.ui.syncTimeNotTrack.isChecked()
        mountTracks = self.app.mount.obsSite.status in [0, 10]
        if doSyncNotTrack and mountTracks:
            return False

        delta = self.app.mount.obsSite.timeDiff * 1000
        if abs(delta) < 10:
            return False

        if delta > 999:
            delta = 999
        if delta < -999:
            delta = -999

        delta = int(delta)
        suc = self.app.mount.obsSite.adjustClock(delta)
        if not suc:
            self.msg.emit(2, 'System', 'Clock',
                          'Cannot adjust mount clock')
            return False

        self.msg.emit(0, 'System', 'Clock',
                      f'Correction: [{-delta} ms]')
        return True

    def updateTelescopeParametersToGui(self):
        """
        updateTelescopeParametersToGui takes the information gathered from the
        driver and programs them into gui for later use.

        :return: true for test purpose
        """

        value = self.app.telescope.data.get(
            'TELESCOPE_INFO.TELESCOPE_FOCAL_LENGTH', 0)
        if value is not None:
            value = float(value)
            self.ui.focalLength.setValue(value)

        value = self.app.telescope.data.get('TELESCOPE_INFO.TELESCOPE_APERTURE',
                                            0)
        if value is not None:
            value = float(value)
            self.ui.aperture.setValue(value)

        return True

    def updateTelescopeParametersToGuiCyclic(self):
        """
        :return:
        """
        if not self.ui.automaticTelescope.isChecked():
            return False
        self.updateTelescopeParametersToGui()
        return True


from beartype.typing import (
    Callable,
    Union,
)

# ....................{ HINTS                              }....................
BeartypeValidatorTester = Callable[[object], bool]
'''
PEP-compliant type hint matching a **beartype validator tester** (i.e.,
caller-defined callable accepting a single arbitrary object and returning
either ``True`` if that object satisfies an arbitrary constraint *or* ``False``
otherwise).

Beartype validator testers are suitable for subscripting functional beartype
validator factories (e.g., :attr:`beartype.vale.Is`).
'''

BeartypeValidatorRepresenter = Union[str, Callable[[], str]]
'''
PEP-compliant type hint matching a **beartype validator representer** (i.e.,
either a string *or* caller-defined callable accepting no arguments returning a
machine-readable representation of a beartype validator).

Technically, this representation *could* be passed by the caller rather than
this callable dynamically generating that representation. Pragmatically,
generating this representation is sufficiently slow for numerous types of
validators that deferring their generation until required by a call to the
:meth:`__repr__` dunder method externally called by a call to the :func:`repr`
builtin` on this validator is effectively mandatory. Validators whose
representations are particularly slow to generate include:

* The :class:`Is` class subscripted by a lambda rather than non-lambda
  function. Generating the representation of that class subscripted by a
  non-lambda function only requires introspecting the name of that function and
  is thus trivially fast. However, lambda functions have no names and are thus
  *only* distinguishable by their source code; generating the representation of
  that class subscripted by a lambda function requires parsing the source code
  of the file declaring that lambda for the exact substring of that code
  declaring that lambda.
'''


from unittest import skip

from django.urls import reverse
from playwright.sync_api import sync_playwright, expect

from bookmarks.e2e.helpers import LinkdingE2ETestCase

class BookmarkItemE2ETestCase(LinkdingE2ETestCase):
    @skip("Fails in CI, needs investigation")
    def test_toggle_notes_should_show_hide_notes(self):
        bookmark = self.setup_bookmark(notes='Test notes')

        with sync_playwright() as p:
            page = self.open(reverse('bookmarks:index'), p)

            notes = self.locate_bookmark(bookmark.title).locator('.notes')
            expect(notes).to_be_hidden()

            toggle_notes = page.locator('li button.toggle-notes')
            toggle_notes.click()
            expect(notes).to_be_visible()

            toggle_notes.click()
            expect(notes).to_be_hidden()


# -*- coding: utf-8 -*-
"""
Stage 3: Key points detection
Author: Jing Ning @ SunLab
"""

import time
import numpy as np
from SoAL_Constants import MODEL_FOLDER, PRINT_FRAME

"""
Queue pack:
[task_id, img, frame, roi_i, {reg}]
reg:
[area, center, orient + 90, major, minor, center_global]
"""
PACK_TASK_ID, PACK_IMG, PACK_FRAME, PACK_ROI_I, PACK_REG, PACK_REG_N = 0, 1, 2, 3, 4, 5
REG_AREA, REG_CENTER, REG_ORIENT90, REG_MAJOR, REG_MINOR, REG_CENTER_GLOBAL = 0, 1, 2, 3, 4, 5
class PredictParts(object):
    def __init__(self, q, i):
        # redirect_output("log/pred%d.log" % i)
        self._pred_q = q
        self._idx = i
        self.stop = False
        self._task_q_l = {}
        self.init_pred_model() #TEST:NEED_PRED

    def init_pred_model(self):
        from hrnet import predict
        self._model = predict.PredictModel(MODEL_FOLDER)

    def append_new_task(self, pack):
        from multiprocessing import Process, Queue
        task_id, video = pack
        q = Queue()
        self._task_q_l[task_id] = q
        from SoAL_IDAssign import IdAssign  # NOTE: diff
        print("[predict_parts] new task %d %s" % (task_id, video))
        rs_p = Process(target=IdAssign.process, args=(video, task_id, q))  # copy instance
        rs_p.start()

    def end_task(self, task_id):
        self._task_q_l[task_id].put(None)

    def predict_loop(self):
        print("[predict_parts%d] loop..."%self._idx)
        count, n = 0, 0
        last_ts = time.time()
        while not self.stop:
            l = self._pred_q.qsize()
            if not l:
                l = 1
                # print("q empty")
            pack_l = []
            img_l = []
            for i in range(l):
                pack = self._pred_q.get()
                task_id = pack[PACK_TASK_ID]
                if task_id not in self._task_q_l:
                    self.append_new_task(pack)
                else:
                    img = pack[PACK_IMG]
                    if img is None:
                        self.end_task(task_id)
                    else:
                        # plot_ego("img/img%06d%08d.jpg" % (pack[PACK_FRAME], i), pack[PACK_IMG], None)
                        # continue
                        img_l.append(img)
                        pack_l.append(pack)
            if not img_l:
                continue
            # for i, img in enumerate(img_l):
            #     print("%d %s"%(i,img.shape))
            ypk, scmap = self._model.pred_img_batch(img_l)
            # ypk = np.zeros((3, 5, len(img_l))) #TEST:NEED_PRED
            # ypk: (3,5,n)  scmap: (n,8,8,5)
            for i, img in enumerate(img_l):
                # print("task", task_id, "batch", len(img_l))
                pack = pack_l[i]
                task_id = pack[PACK_TASK_ID]
                reg = pack[PACK_REG]
                # if pack[PACK_FRAME] >=0 and pack[PACK_ROI_I]==0:
                #     plot_ego("img/img%06d%08d.jpg" % (pack[PACK_FRAME], i), pack[PACK_IMG], np.array(ypk[:, :, i]))  #TEST:PLOT_EGO
                points = rotate_back_points(ypk[:, :, i], reg[REG_CENTER], reg[REG_ORIENT90], img.shape)
                # points: (3,5)
                self._task_q_l[task_id].put([pack[PACK_ROI_I], pack[PACK_FRAME], reg, pack[PACK_REG_N], np.array(points).flatten()])
            count += l
            n += 1
            if count >= PRINT_FRAME*5:
                # print("[predict_parts] pred_img_batch: %f/%d" % (prof_t, prof_c))
                ts = time.time()
                d_ts = ts - last_ts
                last_ts = ts
                print("[predict_parts%d] (%d/%d)fly/q %.2fs %.2ffly/s" % (self._idx, count, n, d_ts, count/d_ts))
                # profile: 1800fly/s = tasks(4)*avg_fps(15)*rois(15)*2
                count, n = 0, 0

    @staticmethod
    def process(q, i=0):
        #import os
        #os.environ["CUDA_VISIBLE_DEVICES"] = "%d" % (i%GPU_N)
        try:
            #     with tf.device("/gpu:%d" % (i%GPU_N)):
            PredictParts.inst = PredictParts(q, i)
            PredictParts.inst.predict_loop()
        except Exception as e:
            import traceback
            traceback.print_exc()

    @staticmethod
    def stop():
        PredictParts.inst.stop = True

def rotate_back_points(ypk0, pos, flydir, img_shape):
    flyx, flyy = pos[0], pos[1]
    fd = np.deg2rad(flydir - 90)  # FIXME version dependent (flydir)
    xy1 = np.vstack((ypk0[0], ypk0[1], np.ones((1, 5))))
    xy1center = np.matrix([
        [-1, 0, img_shape[1] / 2],
        [0, -1, img_shape[0] / 2],
        [0, 0, 1],
    ]) * xy1
    rot_m = np.matrix([
        [np.cos(fd), -np.sin(fd), 0],
        [np.sin(fd), np.cos(fd), 0],
        [0, 0, 1],
    ])
    trans_m = np.matrix([
        [1, 0, flyx],
        [0, 1, flyy],
        [0, 0, 1],
    ])
    m = trans_m * rot_m
    xy1_trans = m * xy1center
    return xy1_trans

def plot_ego(name, img, points):
    import cv2
    import matplotlib.pyplot as plt
    from matplotlib.colors import NoNorm
    cv2.imwrite(name.replace("jpg", "png"), img)
    # return
    xs, ys = points[0], points[1]
    plt.cla()
    plt.imshow(img.astype(int), cmap=plt.cm.gray, norm=NoNorm())
    plt.plot(xs[:3], ys[:3], 0.5, "r")
    plt.plot(xs[1:4:2], ys[1:4:2], 0.5, "g")#1, 3
    plt.plot(xs[1:5:3], ys[1:5:3], 0.5, "b")#1,4
    plt.scatter(xs, ys, c="rgbkw")
    s = img.shape
    plt.xlim((0, s[0]))
    plt.ylim((s[1], 0))
    plt.savefig(name)

{'predict': 'hrnet.predict', 'Process': 'multiprocessing.Process', 'Queue': 'multiprocessing.Queue', 'IdAssign': 'SoAL_IDAssign.IdAssign'}.inst.predict_loop

{'predict': 'hrnet.predict', 'Process': 'multiprocessing.Process', 'Queue': 'multiprocessing.Queue', 'IdAssign': 'SoAL_IDAssign.IdAssign', 'traceback': 'traceback'}.inst


import cv2
import numpy as np
import dlib
import matplotlib
matplotlib.use('TKAgg')
import matplotlib.pyplot as plt
import time
import dlib
import argparse
import os

from expression_prototypes import get_expression_prototypes, get_matching_expression_prototypes
from profiling import PerformanceTimer
from transforms import dlib_rect_to_bb, landmarks_to_points, verts_to_indices, landmark_indices_to_triangles, apply_transform
from processing import get_scaled_rgb_frame, get_face_bb_landmarks, triangulate_landmarks, get_transforms, get_face_coordinates_system, landmarks_to_image_space
from visualisation import view_landmarks, debug_fill_triangles
from transfer import transfer_expression, transfer_face, transfer_poly
from utils import *

parser = argparse.ArgumentParser(description='Facial re-enactment prototype from source to target clip')
parser.add_argument("--source", metavar="/path/to/source.mp4", help="path to source video clip")
parser.add_argument("--target", metavar="/path/to/target.mp4", help="path to target video clip")
parser.add_argument("--output", required=False, metavar="output.mp4", help="output file path, optional")
parser.add_argument("--debugtriangles", required=False, action='store_true', help="specify this to see tracking triangles")
parser.add_argument("--blank", required=False, action='store_true', help="include this to make the resulting face on top of a blank canvas, for easier inspection")
parser.add_argument("--slow", required=False, action='store_true', help="include this to see the face reconstruction in a slow rate for debugging purposes")
parser.add_argument("--srcscale", type=float, required=False, metavar=1, help="Specify scaling of source frames ( use smaller values to improve performance )", default=0.25)
parser.add_argument("--trgscale", type=float, required=False, metavar=1, help="Specify scaling of target frames ( use smaller values to improve performance )", default=0.25)
parser.add_argument("--use3D", required=False, action='store_true', help="Specify to use 3D transfer method instead of 2D")

args = parser.parse_args()

assert args.source, "Argument --source must be provided"
assert args.target, "Argument --target must be provided"

DEBUG_TRIANGLES = args.debugtriangles
generated_colors = False
generated_triangulation = False
colors = []

profiler = PerformanceTimer()

sourceName = args.source
targetName = args.target
outputName = args.output

predictor_data = 'shape_predictor_68_face_landmarks.dat'

source = cv2.VideoCapture(sourceName)
target = cv2.VideoCapture(targetName)
output = None

sourceSuccess = True
targetSuccess = True
frameTime = 1/24

face_detector = dlib.get_frontal_face_detector()
landmark_predictor = dlib.shape_predictor(predictor_data)

fig = plt.figure()

#some offline processing before rendering

#source frame
srcWindow = fig.add_subplot(161)
#target frame
trgWindow = fig.add_subplot(162)
#target prototype
protoWindow = fig.add_subplot(163)
#transformed prototype
transformedProtoWindow = fig.add_subplot(164)

#transformed target frame
transformedTargetWindow = fig.add_subplot(165)
#final result frame
protoToTargetWindow = fig.add_subplot(166)

#get landmark faces in source video using some sort of disimilarity measure

#average sum euclidean distance?
#some others - compare performance?

#performance measure - self re-enacntment contour comparison makes most sense

srcScalingFactor = args.srcscale
trgScalingFactor = args.trgscale

sourceSuccess, sourceFrame = get_scaled_rgb_frame(source, srcScalingFactor)
targetSuccess, targetFrame = get_scaled_rgb_frame(target, trgScalingFactor)
im_h,im_w = sourceFrame.shape[:2]
trg_h, trg_w = targetFrame.shape[:2]

srcPlot = srcWindow.imshow(sourceFrame)
trgPlot = trgWindow.imshow(targetFrame)
transformedTargetPlot = transformedTargetWindow.imshow(targetFrame)
protoPlot = protoWindow.imshow(targetFrame)
transformedProtoPlot = transformedProtoWindow.imshow(targetFrame)
finalPlot = protoToTargetWindow.imshow(targetFrame)

plt.ion()
plt.show()

target_prototype_matching_faces = []

source_prototype_faces, source_frame_prototype_index = get_expression_prototypes(source, srcScalingFactor, profiler)

#go through target video and identify key face frames that  matched source video prototypes
#use similarity measure to also point which reference face should be matched durring time measures

target_prototype_faces_frames_landmarks = get_matching_expression_prototypes(target, trgScalingFactor, source_prototype_faces, profiler)

frameCount = 0
srcLandmarkTrianglesIndices = []

#go

#reopen clips to reset capture
source = cv2.VideoCapture(sourceName)
target = cv2.VideoCapture(targetName)
sourceFPS = source.get(cv2.CAP_PROP_FPS)

base_output_folder = 'debug\\{}'.format(outputName)
if not os.path.exists(base_output_folder):
    os.makedirs(base_output_folder, exist_ok=True)

base_output_path = base_output_folder+'\\{}.jpeg'

while targetSuccess and sourceSuccess:
    use3D = args.use3D
    print("Processing frame {} ...".format(frameCount))
    frameCount = frameCount + 1

    profiler.tick("Process Frame")
    #get frames
    sourceSuccess, sourceFrame = get_scaled_rgb_frame(source, srcScalingFactor)
    targetSuccess, targetFrame = get_scaled_rgb_frame(target, trgScalingFactor)
    if not sourceSuccess or not targetSuccess:
        break

    try:
        target_prototype,target_prototype_landmarks = target_prototype_faces_frames_landmarks[source_frame_prototype_index[frameCount-1]]
    except:
        target_prototype,target_prototype_landmarks = target_prototype_faces_frames_landmarks[source_frame_prototype_index[len(source_frame_prototype_index)-1]]

    srcPlot.set_data(sourceFrame)
    trgPlot.set_data(targetFrame)
    protoPlot.set_data(target_prototype)

    sourceToTarget, newTargetLandmarks, triangle_landmark_indices, source_frame_landmarks, target_landmarks = transfer_expression(sourceFrame, targetFrame, trgWindow, srcWindow, profiler, use3D)

    sourceToProto, newProtoLandmarks, _ , _, _ = transfer_expression(sourceFrame, target_prototype, protoWindow, srcWindow, profiler, use3D)

    #sourceToProto, newProtoLandmarks, _ , _= transfer_expression(sourceFrame, targetFrame, protoWindow, srcWindow, profiler)
    if(sourceToTarget is None or sourceToProto is None):
        continue

    transformedTargetPlot.set_data(sourceToTarget)
    transformedProtoPlot.set_data(sourceToProto)
    protoToSource = transfer_poly(sourceToTarget, sourceToProto, newTargetLandmarks, newProtoLandmarks)
    #protoToSource = transfer_poly(sourceToProto,sourceToTarget, newProtoLandmarks, newTargetLandmarks)

    cloneMask = np.zeros(protoToSource.shape, protoToSource.dtype)
    cloneContour = cv2.convexHull(np.int32(target_landmarks))
    cloneMask = cv2.fillConvexPoly(cloneMask, cloneContour, (255,255,255))
    cloneMask = cv2.erode(cloneMask, np.ones((5,5)))
    #finalFrame = cv2.seamlessClone(protoToSource, np.copy(targetFrame), cloneMask ,(int(trg_w/2),int(trg_h/2)),cv2.NORMAL_CLONE)
    target_face_center, _, _, _, target_local_landmarks  = get_face_coordinates_system(target_landmarks);
    finalFrame = cv2.seamlessClone(protoToSource, np.copy(targetFrame), cloneMask ,(int(target_face_center[0]),int(target_face_center[1]-10)),cv2.NORMAL_CLONE)

   # finalFrame = protoToSource;
    #protoToSource = transfer_face(sourceToProto, sourceToTarget, newProtoLandmarks, newTargetLandmarks, triangle_landmark_indices)
    #protoToSource = transfer_face(sourceFrame,  source_frame_landmarks ,sourceToProto, sourceToTarget, triangle_landmark_indices)
    if args.slow:
        plt.pause(frameTime)

    finalPlot.set_data(finalFrame)

    src_out_path = base_output_path.format('{}_src'.format(frameCount))
    cv2.imwrite(src_out_path, cv2.cvtColor(sourceFrame, cv2.COLOR_RGB2BGR))
    trg_out_path = base_output_path.format('{}_trg'.format(frameCount))
    cv2.imwrite(trg_out_path, cv2.cvtColor(targetFrame, cv2.COLOR_RGB2BGR))
    transformed_target_out_path = base_output_path.format('{}_src2trg'.format(frameCount))
    cv2.imwrite(transformed_target_out_path, cv2.cvtColor(sourceToTarget, cv2.COLOR_RGB2BGR))
    proto_out_path = base_output_path.format('{}_proto'.format(frameCount))
    cv2.imwrite(proto_out_path, cv2.cvtColor(target_prototype, cv2.COLOR_RGB2BGR))
    transformed_proto_out_path = base_output_path.format('{}_src2proto'.format(frameCount))
    cv2.imwrite(transformed_proto_out_path, cv2.cvtColor(sourceToProto, cv2.COLOR_RGB2BGR))
    final_out_path = base_output_path.format('{}_result'.format(frameCount))
    cv2.imwrite(final_out_path, cv2.cvtColor(finalFrame, cv2.COLOR_RGB2BGR))

    if args.output and output is None:
        output = cv2.VideoWriter(args.output, cv2.VideoWriter_fourcc('m', 'j', 'p', 'g'), sourceFPS, (finalFrame.shape[1], finalFrame.shape[0]))
    if output:
        output.write(np.uint8(cv2.cvtColor(finalFrame, cv2.COLOR_RGB2BGR)))
    print("Done one full frame process")
    profiler.tock()

    plt.pause(frameTime)
    for p in reversed(srcWindow.patches):
        p.remove()
    for p in reversed(trgWindow.patches):
        p.remove()
    for p in reversed(protoWindow.patches):
        p.remove()
    for p in reversed(transformedProtoWindow.patches):
        p.remove()

print('Transfer complete')
output.release()
source.release()
target.release()


__author__ 		= "Lekan Molu"
__copyright__ 	= "2021, Hamilton-Jacobi Analysis in Python"
__credits__  	= "There are None."
__license__ 	= "MIT License"
__maintainer__ 	= "Lekan Molu"
__email__ 		= "patlekno@icloud.com"
__status__ 		= "Completed"

import copy, os
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm
from os.path import join
from LevelSetPy.Utilities import error
from .color_utils import cmaps
from mpl_toolkits.mplot3d import Axes3D

def buffered_axis_limits(amin, amax, buffer_factor=1.0):
    """
    Increases the range (amin, amax) by buffer_factor on each side
    and then rounds to precision of 1/10th min or max.
    Used for generating good plotting limits.
    For example (0, 100) with buffer factor 1.1 is buffered to (-10, 110)
    and then rounded to the nearest 10.
    """
    diff = amax - amin
    amin -= (buffer_factor-1)*diff
    amax += (buffer_factor-1)*diff
    magnitude = np.floor(np.log10(np.amax(np.abs((amin, amax)) + 1e-100)))
    precision = np.power(10, magnitude-1)
    amin = np.floor(amin/precision) * precision
    amax = np.ceil (amax/precision) * precision
    return (amin, amax)

class Visualizer():

    def __init__(self, fig=None, ax=None, winsize=None,
                labelsize=18, linewidth=6, fontdict=None,
                block=False, savedict=None):
        """
            Ad-hoc visualizer for Grids, and grid partitions

            fig: pyplot figure. If not passed, it's created
            ax: subfig of fig on which to plot figure
            winsize: size of pyplot window (default is (16,9))
            labelsize: size of plot x/y labels
            linewidth: width of 2D lines
            fontdict: fontweight and size for visualization
            block: whether to block screen after plot or not

            Author: Lekan MOlux, August/September 2021
        """
        if winsize is None:
            self.winsize =(16, 9)
        self.linewidth = linewidth
        if (fig and ax) is None:
            self._fig = plt.figure(figsize=winsize)
            self._fig.tight_layout()
        self.block=block

        self._labelsize = labelsize
        self._fontdict  = fontdict
        self._projtype = 'rectilinear'
        self.savedict = savedict

        if self.savedict["save"] and not os.path.exists(self.savedict["savepath"]):
            os.makedirs(self.savedict["savepath"])

        if self._fontdict is None:
            self._fontdict = {'fontsize':12, 'fontweight':'bold'}

    def visGrid(self, gs, dim, colors=None, save_dir=None, title=None, dims=None):
        # see helper OC/visualization/visGrid.m
        if not colors:
            colors = ['blue', 'red', 'yellow', 'orange', 'green', 'black']

        if dim==1:
            g = gs
            ax = self._fig.add_subplot(1, 1, 1)
            ax.plot(np.zeros((g.N, 1)), g.vs[0], color=colors[0], linestyle='.')
            ax.plot(np.hstack([g.min, g.max]), np.hstack([min(g.vs[0]), max(g.vs[0])]), \
                    linestyle='-', color=colors[0])
            ax.xaxis.set_tick_params(labelsize=self._labelsize)
            ax.yaxis.set_tick_params(labelsize=self._labelsize)
            if title:
                ax.set_title(title, fontdict=self._fontdict)

            if dims:
                x, y = dims[i]
                ax.annotate(f'dim={dims[i]}', xy=(x+0.25, y-0.5), size=17)

            self._fig.tight_layout()
            # if save_dir:
            #     self.fig.savefig(join(save_dir, datetime.strftime(datetime.now() + '%H-%')+'.png'))
        elif dim==2:
            ax = self._fig.add_subplot(1, 1, 1)

            title=f'A {len(gs)}-cell/{gs[0].dim}D-Grid'
            for i in range(len(gs)):
                g = gs[i]
                ax.plot(g.xs[0], g.xs[1], '.', color=colors[i])
                ax.plot(np.hstack([g.min[0], g.min[0]]), np.hstack([g.min[1], g.max[1]]), linestyle='-', color=colors[i])
                ax.plot(np.hstack([g.max[0], g.max[0]]), np.hstack([g.min[1], g.max[1]]), linestyle='-', color=colors[i])
                ax.plot(np.hstack([g.min[0], g.max[0]]), np.hstack([g.min[1], g.min[1]]), linestyle='-', color=colors[i])
                ax.plot(np.hstack([g.min[0], g.max[0]]), np.hstack([g.max[1], g.max[1]]), linestyle='-', color=colors[i])

                ax.xaxis.set_tick_params(labelsize=self._labelsize)
                ax.yaxis.set_tick_params(labelsize=self._labelsize)

                ax.set_xlabel('x', fontdict=self._fontdict)
                ax.set_ylabel('y', fontdict=self._fontdict)

                if dims:
                    x, y = dims[i]
                    ax.annotate(f'dim={dims[i]}', xy=(x+0.25, y-0.5), size=17)

            if not title:
                title = f'Gridsplitter along {len(gs)} dims'

            plt.title(title, fontdict=self._fontdict)
        elif dim==3:
            ax = self._fig.add_subplot(1, 2, 1, projection='3d')

            # title=f'A {len(gs)}-cell/{gs[0].dim}D-Grid'
            g = gs
            g.xs[0] = g.xs[0].reshape(np.prod(g.xs[0].shape), 1)
            g.xs[1] = g.xs[1].reshape(np.prod(g.xs[1].shape), 1)
            g.xs[2] = g.xs[2].reshape(np.prod(g.xs[2].shape), 1)

            ax.plot_wireframe(g.xs[0], g.xs[1], g.xs[2], rstride=10, cstride=10)

            ax.set_xlabel('x', fontdict=self._fontdict)
            ax.set_ylabel('y', fontdict=self._fontdict)
            ax.set_zlabel('z', fontdict=self._fontdict)

            ax.xaxis.set_tick_params(labelsize=self._labelsize)
            ax.yaxis.set_tick_params(labelsize=self._labelsize)
            ax.zaxis.set_tick_params(labelsize=self._labelsize)

            ax = self._fig.add_subplot(1, 2, 2, projection='3d')
            surf = ax.plot_surface(g.xs[0], g.xs[1], g.xs[2],rstride=1, cstride=1, cmap=cm.coolwarm,
                                   linewidth=0, antialiased=False)

            plt.title(title, fontdict=self._fontdict)
            self._fig.colorbar(surf, shrink=0.5, aspect=10)

        elif dim>3: # this is for a projected grid to 2d
            ax = self._fig.add_subplot(1, 1, 1)
            i=0
            for g in gs:
                ax.plot(g.xs[0], g.xs[1], '.', color=colors[i])
                ax.plot(np.hstack([g.min[0], g.min[0]]), np.hstack([g.min[1], g.max[1]]), linestyle='-', color=colors[i])
                ax.plot(np.hstack([g.max[0], g.max[0]]), np.hstack([g.min[1], g.max[1]]), linestyle='-', color=colors[i])
                ax.plot(np.hstack([g.min[0], g.max[0]]), np.hstack([g.min[1], g.min[1]]), linestyle='-', color=colors[i])
                ax.plot(np.hstack([g.min[0], g.max[0]]), np.hstack([g.max[1], g.max[1]]), linestyle='-', color=colors[i])

                ax.xaxis.set_tick_params(labelsize=self._labelsize)
                ax.yaxis.set_tick_params(labelsize=self._labelsize)

                ax.set_xlabel('x', fontdict=self._fontdict)
                ax.set_ylabel('y', fontdict=self._fontdict)
                if dims:
                    x, y = dims[i]
                    ax.annotate(f'dim={dims[i]}', xy=(x+0.25, y-0.5), size=17)
                i+=1
            title=f'A {len(gs)}-cell/{g.dim}D-Grid'
            self._fig.tight_layout()
            plt.title(title, fontdict=self._fontdict)
        else:
            error('Only Grids of up to 3 dimensions can be visualized!')

        if self.savedict["save"]:
            plt.savefig(join(self.savedict["savepath"],self.savedict["savename"]),
                        bbox_inches='tight', facecolor='None')

        plt.show(block=self.block)

    def visFuncIm(self, g, dataPlot,color):
        ax = self._fig.add_subplot(1, 1, 1)
        if g.dim<2:
            ax.plot(g.xs[0], np.squeeze(dataPlot), linewidth=2, color = color)
        elif g.dim==2:
            surf = ax.plot_surface(g.xs[0], g.xs[1], dataPlot, \
                                   rstride=1, cstride=1, cmap=cm.coolwarm,
                                   linewidth=0, antialiased=False, facecolors=color)
        else:
            error('Can not plot in more than 3D!')

        plt.show(block=self.block)


from datetime import datetime

import requests
import json
import csv

def csv_to_json(csv_path, json_path, host, port, collection):
    url = f"http://{host}:{port}/{collection}"
    jsonArray = []

    with open(csv_path, encoding='utf-8') as csv_file:
        csvReader = csv.DictReader(csv_file)

        for row in csvReader:
            if collection == 'flights':
                date_str = row['date']
                date_obj = datetime.strptime(date_str, '%d/%m/%Y').date()

                row['date'] = date_obj.strftime('%Y-%m-%d')

            elif collection == 'offers':
                date_str_start = row['start_date']
                date_str_end = row['end_date']
                date_obj_start = datetime.strptime(date_str_start, '%d/%m/%Y').date()
                date_obj_send = datetime.strptime(date_str_end, '%d/%m/%Y').date()

                row['start_date'] = date_obj_start.strftime('%Y-%m-%d')
                row['end_date'] = date_obj_send.strftime('%Y-%m-%d')

            jsonArray.append(row)
            payload = json.dumps(row)

            headers = {
                'Content-Type': 'application/json'
            }
            response = requests.request("POST", url, headers=headers, data=payload)
            print(response.text)

    with open(json_path, 'w', encoding='utf-8') as json_file:
        jsonString = json.dumps(jsonArray, indent=4, ensure_ascii=False)
        json_file.write(jsonString)

offers_csv_path = r'hoteleOfertyBaza.csv'
offers_json_path = r'offers.json'
# csv_to_json(offers_csv_path, offers_json_path, "localhost", 8085, "offers")

flights_csv_path = r'lotyOfertyBaza.csv'
flights_json_path = r'flights.json'
# csv_to_json(flights_csv_path, flights_json_path, "localhost", 8085, "flights") # 5245 cluster 8085 local


import threading
import logging
import os
import json
import re
import time

FREQ_RE = re.compile("^(\d+)([smhd])$")
SUFFIX_TO_SEC_MULTIPLIER = {"s": 1, "m": 60, "h": 60 * 60, "d": 60 * 60 * 24}

def parse_freq(s_freq):
    m = FREQ_RE.match(s_freq)
    return int(m.group(1)) * SUFFIX_TO_SEC_MULTIPLIER[m.group(2)]

class MonitorManager(threading.Thread):
    def __init__(self, block, unix_service):
        threading.Thread.__init__(self)
        self.id = block["id"]
        self.unix_service = unix_service
        self.block = block

    def run(self):
        freq = parse_freq(self.block.get("frequency", "1m"))
        logging.info(
            "[{}] Running {} every {} seconds".format(
                self.block["id"], self.block["command"], freq
            )
        )
        logging.debug("[{}] 5 seconds for startup".format(self.block["id"]))
        time.sleep(5)
        while True:
            result = self.unix_service.run_block(self.block, (), None)
            if isinstance(result, str):
                result = result.strip()

            logging.debug("[{}] Result: {}".format(self.block["id"], result))
            self.process(result)
            time.sleep(freq)

    def process(self, data):
        self.unix_service.emit_event(self.id, data)


import sys
# general + data loading
import pandas as pd
from sqlalchemy import create_engine

# pre-processing
from nltk import pos_tag
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import re

# machine learning
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.multioutput import MultiOutputClassifier
#from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

# storage
import pickle

# Load custom functions from the model folder
import sys, os
sys.path.insert(1, os.path.join(os.path.dirname(os.path.realpath(__file__))))
import glove
import ml_helper as utils

def load_data(database_filepath):
  '''Loads the relevant data for the model training.'''
  # download glove data
  if not glove.download('twitter'):
    print("ERROR: Download of GloVe Data failed. Please download and extract manually to the ./models directory from http://nlp.stanford.edu/data/wordvecs/glove.twitter.27B.zip")
  # laad remaining data
  engine = create_engine('sqlite:///{}'.format(database_filepath))
  df = pd.read_sql_table('texts', engine)
  X = df[['message', 'original', 'genre']]
  Y = df.iloc[:, 4:]
  return X['message'], Y, Y.columns

def build_model():
  '''Creates the pipeline with pre-processing and classifier.'''
  pipeline = Pipeline([
    ('features', FeatureUnion([
      ('term_emb', Pipeline([
        ('vectorize', CountVectorizer(tokenizer=utils.tokenize_clean, max_df=0.5)),
        ('tfidf', TfidfTransformer(use_idf=False)),
      ])),
      ('glove', Pipeline([
        ('glove_emb', glove.GloVeTransformer('twitter', 25, 'centroid', tokenizer=utils.tokenize_clean, max_feat=5))
      ]))
    ])),
    ('cls', MultiOutputClassifier(LogisticRegression(max_iter=50, C=2.0), n_jobs=-1) )
  ])
  return pipeline

def evaluate_model(model, X_test, Y_test, category_names):
  '''Evaluate the model.'''
  utils.score_and_doc(model, "deploy_final", X_test, Y_test, extended=True)

def save_model(model, model_filepath):
  '''Store the model as serialized data.'''
  pickle.dump(model, open(model_filepath, 'wb'))

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
from requests import Request, Session, Timeout
import dateutil.parser
from datetime import datetime
from rest_framework.exceptions import ValidationError

from django.core.cache import cache
from django.conf import settings
from django.contrib.gis.geos import Point, Polygon
from django.utils.timezone import make_aware

from apps.bigcz.models import ResourceLink, ResourceList
from apps.bigcz.utils import RequestTimedOutError

from apps.bigcz.clients.hydroshare.models import HydroshareResource

CATALOG_NAME = 'hydroshare'
CATALOG_URL = 'https://www.hydroshare.org/hsapi/resource/'

DATE_MIN = make_aware(datetime(1776, 7, 4, 0, 0))

def parse_date(value):
    d = dateutil.parser.parse(value)
    if d.tzinfo is None or d.tzinfo.utcoffset(d) is None:
        return make_aware(d)
    else:
        return d

def parse_box(box):
    return Polygon((
        (float(box['westlimit']), float(box['northlimit'])),
        (float(box['eastlimit']), float(box['northlimit'])),
        (float(box['eastlimit']), float(box['southlimit'])),
        (float(box['westlimit']), float(box['southlimit'])),
        (float(box['westlimit']), float(box['northlimit'])),
    ))

def parse_point(point):
    return Point(float(point['east']), float(point['north']))

def parse_geom(coverages):
    geoms = []
    if coverages:
        geoms.extend([parse_box(c['value'])
                      for c in coverages
                      if c['type'] == 'box'])
        geoms.extend([parse_point(c['value'])
                      for c in coverages
                      if c['type'] == 'point'])
        if geoms:
            geom = geoms[0]
            for g in geoms[1:]:
                geom |= g

            return geom

    return None

def parse_coverage_period(coverages):
    if coverages:
        period = [c['value'] for c in coverages if c['type'] == 'period']
        if period and period[0]:
            return period[0]['start'], period[0]['end']

    return None, None

def parse_record(item):
    start, end = parse_coverage_period(item['coverages'])

    return HydroshareResource(
        id=item['resource_id'],
        title=item['resource_title'],
        description=None,
        author=item['creator'],
        links=[
            ResourceLink('details', item['resource_url'])
        ],
        created_at=parse_date(item['date_created']),
        updated_at=parse_date(item['date_last_updated']),
        geom=parse_geom(item['coverages']),
        begin_date=parse_date(start) if start else None,
        end_date=parse_date(end) if end else None)

def prepare_bbox(box):
    return {
        'west': box.xmin,
        'east': box.xmax,
        'north': box.ymin,
        'south': box.ymax,
    }

def prepare_date(value):
    return value.strftime('%Y-%m-%d')

def nullable_attrgetter(key, default):
    """To allow sorting on nullable field with default value"""
    def getter(item):
        return getattr(item, key) or default

    return getter

def search(**kwargs):
    query = kwargs.get('query')
    to_date = kwargs.get('to_date')
    from_date = kwargs.get('from_date')
    bbox = kwargs.get('bbox')
    page = kwargs.get('page')
    exclude_private = 'exclude_private' in kwargs.get('options')

    if not query:
        raise ValidationError({
            'error': 'Required argument: query'})

    params = {
        'full_text_search': query,
    }

    if bbox:
        params.update(prepare_bbox(bbox))
        params.update({
            'coverage_type': 'box'
        })
    if page:
        params.update({
            'page': page
        })

    session = Session()
    request = session.prepare_request(Request('GET',
                                              CATALOG_URL,
                                              params=params))

    key = f'bigcz_hydroshare_{hash(frozenset(params.items()))}'
    cached = cache.get(key)
    if cached:
        data = cached

    else:
        try:
            response = session.send(request,
                                    timeout=settings.BIGCZ_CLIENT_TIMEOUT)
            data = response.json()
            cache.set(key, data, timeout=1800)  # Cache for half hour
        except Timeout:
            raise RequestTimedOutError()

    if 'results' not in data:
        raise ValueError(data)

    items = data['results']
    if exclude_private:
        items = [item for item in items if item['public']]

    records = [parse_record(item) for item in items]
    # Include only those with geometries
    records = [r for r in records if r.geom]

    if from_date:
        records = [r for r in records
                   if r.end_date and r.end_date >= make_aware(from_date)]

    if to_date:
        records = [r for r in records
                   if r.begin_date and r.begin_date <= make_aware(to_date)]

    results = sorted(records,
                     key=nullable_attrgetter('end_date', DATE_MIN),
                     reverse=True)
    count = data['count']

    return ResourceList(
        api_url=request.url,
        catalog=CATALOG_NAME,
        count=count,
        results=results)


from enum import Enum

from pyspark import SparkContext

from geospark.core.SpatialRDD import SpatialRDD, PolygonRDD, LineStringRDD, PointRDD
from geospark.utils.decorators import require

class DiscLoader:

    @classmethod
    def load(cls, sc: SparkContext, path: str) -> SpatialRDD:
        raise NotImplementedError()

class IndexDiscLoader:

    @classmethod
    def load(cls, sc: SparkContext, path: str):
        jvm = sc._jvm
        index_rdd = jvm.ObjectSpatialRDDLoader.loadIndexRDD(sc._jsc, path)
        return index_rdd

class PolygonRDDDiscLoader(DiscLoader):

    @classmethod
    @require(["ObjectSpatialRDDLoader"])
    def load(cls, sc: SparkContext, path: str) -> SpatialRDD:
        jvm = sc._jvm
        polygon_rdd = PolygonRDD()
        srdd = jvm.ObjectSpatialRDDLoader.loadPointSpatialRDD(sc._jsc, path)
        polygon_rdd.set_srdd(srdd)
        return polygon_rdd

class PointRDDDiscLoader(DiscLoader):

    @classmethod
    @require(["ObjectSpatialRDDLoader"])
    def load(cls, sc: SparkContext, path: str) -> SpatialRDD:
        jvm = sc._jvm
        polygon_rdd = PointRDD()
        srdd = jvm.ObjectSpatialRDDLoader.loadPointSpatialRDD(sc._jsc, path)
        polygon_rdd.set_srdd(srdd)
        return polygon_rdd

class LineStringRDDDiscLoader(DiscLoader):

    @classmethod
    @require(["ObjectSpatialRDDLoader"])
    def load(cls, sc: SparkContext, path: str) -> SpatialRDD:
        jvm = sc._jvm
        line_string_rdd = LineStringRDD()
        srdd = jvm.ObjectSpatialRDDLoader.loadLineStringSpatialRDD(sc._jsc, path)
        line_string_rdd.set_srdd(srdd)
        return line_string_rdd

class SpatialRDDDiscLoader(DiscLoader):

    @classmethod
    @require(["ObjectSpatialRDDLoader"])
    def load(cls, sc: SparkContext, path: str) -> SpatialRDD:
        jvm = sc._jvm
        spatial_rdd = SpatialRDD()
        srdd = jvm.ObjectSpatialRDDLoader.loadSpatialRDD(sc._jsc, path)
        spatial_rdd.set_srdd(srdd)
        return spatial_rdd

class GeoType(Enum):
    POINT = "POINT"
    POLYGON = "POLYGON"
    LINESTRING = "LINESTRING"
    GEOMETRY = "GEOMETRY"

loaders = {
    GeoType.POINT: PointRDDDiscLoader,
    GeoType.POLYGON: PolygonRDDDiscLoader,
    GeoType.LINESTRING: LineStringRDDDiscLoader,
    GeoType.GEOMETRY: SpatialRDDDiscLoader
}

def load_spatial_rdd_from_disc(sc: SparkContext, path: str, geometry_type: GeoType):
    """

    :param sc:
    :param path:
    :param geometry_type:
    :return:
    """
    return loaders[geometry_type].load(sc, path)

def load_spatial_index_rdd_from_disc(sc: SparkContext, path: str):

    return IndexDiscLoader.load(sc, path)


import json
import logging
import uuid
import sys
import errno
import os
import pandas as pd
import seaborn as sns
import traceback
import shutil
import scipy.cluster.vq as vq
from matplotlib import pyplot as plt
from plotly.offline import plot
import plotly.express as px
import plotly.graph_objs as go
from sklearn.decomposition import PCA

from installed_clients.DataFileUtilClient import DataFileUtil
from installed_clients.KBaseReportClient import KBaseReport

class KmeansClusteringUtil:

    def _mkdir_p(self, path):
        """
        _mkdir_p: make directory for given path
        """
        if not path:
            return
        try:
            os.makedirs(path)
        except OSError as exc:
            if exc.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise

    def _validate_run_kmeans_cluster_params(self, params):
        """
        _validate_run_kmeans_cluster_params:
                validates params passed to run_kmeans_cluster method
        """

        logging.info('start validating run_kmeans_cluster params')

        # check for required parameters
        for p in ['matrix_ref', 'workspace_id', 'cluster_set_name', 'k_num']:
            if p not in params:
                raise ValueError('"{}" parameter is required, but missing'.format(p))

    def _gen_clusters(self, clusters, conditionset_mapping):
        clusters_list = list()

        for cluster in list(clusters.values()):
            labeled_cluster = {}
            labeled_cluster.update({'id_to_data_position': cluster})
            if conditionset_mapping:
                id_to_condition = {k: v for k, v in list(conditionset_mapping.items()) if k in list(cluster.keys())}
                labeled_cluster.update({'id_to_condition': id_to_condition})

            clusters_list.append(labeled_cluster)

        return clusters_list

    def _build_kmeans_cluster_set(self, clusters, cluster_set_name, matrix_ref,
                                  conditionset_mapping, conditionset_ref, workspace_id,
                                  clustering_parameters):
        """
        _build_kmeans_cluster_set: build KBaseExperiments.ClusterSet object
        """

        logging.info('start saving KBaseExperiments.ClusterSet object')

        clusters_list = self._gen_clusters(clusters, conditionset_mapping)

        cluster_set_data = {'clusters': clusters_list,
                            'clustering_parameters': clustering_parameters,
                            'original_data': matrix_ref,
                            'condition_set_ref': conditionset_ref}

        cluster_set_data = {k: v for k, v in list(cluster_set_data.items()) if v}

        object_type = 'KBaseExperiments.ClusterSet'
        save_object_params = {
            'id': workspace_id,
            'objects': [{'type': object_type,
                         'data': cluster_set_data,
                         'name': cluster_set_name}]}

        dfu_oi = self.dfu.save_objects(save_object_params)[0]
        cluster_set_ref = str(dfu_oi[6]) + '/' + str(dfu_oi[0]) + '/' + str(dfu_oi[4])

        return cluster_set_ref

    def _build_kmeans_cluster(self, data_matrix_df, k_num):
        """
        _build_kmeans_cluster: Build Kmeans cluster
        """

        logging.info('start building clusters')

        data_matrix_df.fillna(0, inplace=True)
        values = data_matrix_df.values
        rows = data_matrix_df.index

        # normalize observations
        logging.info('start normalizing raw data')
        whiten_values = vq.whiten(values)

        # run kmeans algorithm
        logging.info('start performing Kmeans algorithm')
        centroid, idx = vq.kmeans2(whiten_values, k_num, minit='points')

        clusters = {}
        for list_index, value in enumerate(idx):
            cluster = clusters.get(value)
            if not cluster:
                clusters.update({value: {rows[list_index]: list_index}})
            else:
                cluster.update({rows[list_index]: list_index})

        return clusters, idx

    def _generate_pairplot(self, data_df, cluster_labels):

        output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
        self._mkdir_p(output_directory)

        col = data_df.columns

        if len(col) > 20:
            logging.info('start building PCA plot')
            pacplot_path = os.path.join(output_directory, 'pcaplot.html')
            n_components = 2
            s_values = data_df.values
            pca = PCA(n_components=n_components, whiten=True)
            principalComponents = pca.fit_transform(s_values)

            col = list()
            for i in range(n_components):
                col.append('principal_component_{}'.format(i+1))

            rotation_matrix_df = pd.DataFrame(data=principalComponents,
                                              columns=col,
                                              index=data_df.index)
            rotation_matrix_df['cluster'] = [str(cluster_label) for cluster_label in cluster_labels]

            traces = list()
            for name in set(rotation_matrix_df.cluster):
                trace = go.Scatter(
                    x=list(rotation_matrix_df.loc[rotation_matrix_df['cluster'].eq(name)]['principal_component_1']),
                    y=list(rotation_matrix_df.loc[rotation_matrix_df['cluster'].eq(name)]['principal_component_2']),
                    mode='markers',
                    name=name,
                    text=list(rotation_matrix_df.loc[rotation_matrix_df['cluster'].eq(name)].index),
                    textposition='bottom center',
                    marker=go.Marker(size=10, opacity=0.8, line=go.Line(color='rgba(217, 217, 217, 0.14)',
                                                                        width=0.5)))
                traces.append(trace)

            data = go.Data(traces)
            layout = go.Layout(xaxis=go.XAxis(title='Principal Component 1', showline=False),
                               yaxis=go.YAxis(title='Principal Component 2', showline=False))
            fig = go.Figure(data=data, layout=layout)
            fig.update_layout(legend_title=' Cluster ')

            plot(fig, filename=pacplot_path)
            return pacplot_path

        data_df['cluster'] = [str(cluster_label) for cluster_label in cluster_labels]
        try:
            logging.info('start building pairplot')
            pairplot_path = os.path.join(output_directory, 'pairplot.html')
            fig = px.scatter_matrix(data_df, dimensions=list(col), color='cluster',
                                    symbol='cluster')
            plot(fig, filename=pairplot_path)
        except Exception:
            logging.warning('failed to run plotly pairplot')
            logging.warning(traceback.format_exc())
            logging.warning(sys.exc_info()[2])
            pairplot_path = None

        if not pairplot_path:
            try:
                pairplot_path = os.path.join(output_directory, 'pairplot.png')
                sns_plot = sns.pairplot(data_df, hue='cluster', height=2.5, vars=list(col))
                sns_plot.savefig(pairplot_path)
            except Exception:
                logging.warning('failed to run seaborn pairplot')
                logging.warning(traceback.format_exc())
                logging.warning(sys.exc_info()[2])
                pairplot_path = None

        return pairplot_path

    def _generate_pairplot_content(self, pairplot_path, output_directory, dimension):

        pairplot_content = ''''''

        if pairplot_path:
            pairplot_name = dimension + '_' + os.path.basename(pairplot_path)
            shutil.copy2(pairplot_path,
                         os.path.join(output_directory, pairplot_name))

            if pairplot_name.endswith('.html'):
                pairplot_content += '''\n'''.format(
                                                                                pairplot_name)
            elif pairplot_name.endswith('.png'):
                pairplot_content += '''\n\n'''
            else:
                raise ValueError('Unexpected cluster pairplot file format')
        else:
            pairplot_content += '''\n
'''
            pairplot_content += '''Pairplot is too large to be displayed.
\n'''

        return pairplot_content

    def _generate_cluster_info_content(self, cluster_labels):
        cluster_info = ''''''
        cluster_info += '''\n

                                    Cluster Index
                                    Size

\n'''

        unique_labels = list(set(cluster_labels))
        unique_labels.sort(key=float)
        for index in unique_labels:
            cluster_info += '''\n

                                    {}
                                    {}

\n'''.format(index, cluster_labels.tolist().count(index))

        cluster_info += '''\n
\n'''

        return cluster_info

    def _generate_kmeans_html_report(self, data_matrix_df, cluster_labels, dimension):

        logging.info('start generating html report')
        html_report = list()

        output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
        self._mkdir_p(output_directory)
        result_file_path = os.path.join(output_directory, 'hier_report.html')

        pairplot = self._generate_pairplot(data_matrix_df, cluster_labels)

        cluster_info = self._generate_cluster_info_content(cluster_labels)
        pairplot_content = self._generate_pairplot_content(pairplot, output_directory, dimension)

        with open(result_file_path, 'w') as result_file:
            with open(os.path.join(os.path.dirname(__file__), 'kmeans_report_template.html'),
                      'r') as report_template_file:
                report_template = report_template_file.read()
                report_template = report_template.replace('
Cluster_Info
',
                                                          cluster_info)
                report_template = report_template.replace('
Pairplot
',
                                                          pairplot_content)
                result_file.write(report_template)

        report_shock_id = self.dfu.file_to_shock({'file_path': output_directory,
                                                  'pack': 'zip'})['shock_id']

        html_report.append({'shock_id': report_shock_id,
                            'name': os.path.basename(result_file_path),
                            'label': os.path.basename(result_file_path),
                            'description': 'HTML summary report for Cluster App'
                            })
        return html_report

    def _generate_kmeans_cluster_report(self, cluster_set_ref, workspace_id,
                                        data_matrix_df, cluster_labels, dimension):
        """
        _generate_kmeans_cluster_report: generate summary report
        """

        objects_created = []
        objects_created.append({'ref': cluster_set_ref,
                                'description': 'Kmeans ClusterSet'})

        output_html_files = self._generate_kmeans_html_report(data_matrix_df,
                                                              cluster_labels,
                                                              dimension)

        report_params = {'message': '',
                         'workspace_id': workspace_id,
                         'objects_created': objects_created,
                         'html_links': output_html_files,
                         'direct_html_link_index': 0,
                         'html_window_height': 333,
                         'report_object_name': 'run_kmeans_cluster_' + str(uuid.uuid4())}

        kbase_report_client = KBaseReport(self.callback_url, token=self.token)
        output = kbase_report_client.create_extended_report(report_params)

        report_output = {'report_name': output['name'], 'report_ref': output['ref']}

        return report_output

    def __init__(self, config):
        self.callback_url = config['SDK_CALLBACK_URL']
        self.token = config['KB_AUTH_TOKEN']
        self.scratch = config['scratch']

        # helper kbase module
        self.dfu = DataFileUtil(self.callback_url)

        logging.basicConfig(format='%(created)s %(levelname)s: %(message)s',
                            level=logging.INFO)

        plt.switch_backend('agg')
        sys.setrecursionlimit(150000)

    def run_kmeans_cluster(self, params):
        """
        run_kmeans_cluster: generates Kmeans clusters for Matrix data object

        matrix_ref: Matrix object reference
        workspace_id: the id of the workspace
        cluster_set_name: KBaseExperiments.ClusterSet object name
        dimension: run cluster algorithm on dimension, col or row
        k_num: number of clusters to form

        Optional arguments:
        dist_metric: The distance metric to use. Default set to 'euclidean'.
                   The distance function can be
                   ["braycurtis", "canberra", "chebyshev", "cityblock", "correlation", "cosine",
                    "dice", "euclidean", "hamming", "jaccard", "kulsinski", "matching",
                    "rogerstanimoto", "russellrao", "sokalmichener", "sokalsneath", "sqeuclidean",
                    "yule"]
                   Details refer to:
                   https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.pdist.html

        return:
        cluster_set_ref: KBaseExperiments.ClusterSet object reference
        report_name: report name generated by KBaseReport
        report_ref: report reference generated by KBaseReport
        """

        logging.info('--->\nrunning run_kmeans_cluster\n' +
                     'params:\n{}'.format(json.dumps(params, indent=1)))

        self._validate_run_kmeans_cluster_params(params)

        matrix_ref = params.get('matrix_ref')
        workspace_id = params.get('workspace_id')
        cluster_set_name = params.get('cluster_set_name')
        dimension = params.get('dimension', 'col')
        k_num = params.get('k_num')

        if dimension not in ['col', 'row']:
            raise ValueError('Please use "col" or "row" for input dimension')

        matrix_data = self.dfu.get_objects({'object_refs': [matrix_ref]})['data'][0]['data']

        matrix_data_values = matrix_data['data']
        data_matrix_df = pd.DataFrame(matrix_data_values['values'],
                                      index=matrix_data_values['row_ids'],
                                      columns=matrix_data_values['col_ids'])
        if dimension == 'col':
            data_matrix_df = data_matrix_df.T

        (kmeans_clusters,
         cluster_labels) = self._build_kmeans_cluster(data_matrix_df, k_num)

        clustering_parameters = {'dimension': dimension,
                                 'k_num': str(k_num)}

        cluster_set_ref = self._build_kmeans_cluster_set(
                                        kmeans_clusters,
                                        cluster_set_name,
                                        matrix_ref,
                                        matrix_data.get('{}_mapping'.format(dimension)),
                                        matrix_data.get('{}_conditionset_ref'.format(dimension)),
                                        workspace_id,
                                        clustering_parameters)

        returnVal = {'cluster_set_ref': cluster_set_ref}

        report_output = self._generate_kmeans_cluster_report(cluster_set_ref, workspace_id,
                                                             data_matrix_df, cluster_labels,
                                                             dimension)

        returnVal.update(report_output)

        return returnVal


import random
import pandas
import sys
import pyomo.environ as pyo
from pyomo.opt import SolverFactory
import csv
import statistics as stat

#config file containing location of graph and discount function csv files
config_file = open("./data_files/email.config").readlines()
#config_file = open("./data_files/redditunif-negxsquared.config").readlines()
#config_file = open("./data_files/astrolaptop-negxsquared.config").readlines()

def ProbFunc(nodeType,c):
	#return 2*c-c*c

	if nodeType==1:
		return c*c
	elif nodeType==2:
		return c
	else:
		return (2-c)*c

#Set default values for filepaths and running variables
data_csv=""
func_csv=""
output_dir=""

#Note that we do not currently use start, end, DStep, or batches

data_counter=1
for line in config_file:
	if str(line)[0] =="#":
		next
	else:
		if data_counter==1:
			data_csv=line[:-1]
		elif data_counter==2:
			func_csv=line[:-1]
		elif data_counter==3:
			output_dir=line[:-1]
		data_counter+=1

#Graph Creation
temp=pandas.read_csv(data_csv,sep=",",header=None)

numV = int(temp[0][0].split()[0])
numE = int(temp[0][0].split()[1])

adj={}
radj={}
P={}
rP={}

for lines in temp[0]:
	line=lines.split()
	if len(line)<3:
		continue
	u=int(line[0])
	v=int(line[1])
	deg=int(line[2])
	#edge probabilites are calculated using the degree
	pp=1.0/deg

	if u in adj.keys():
		adj[u].append((u,v))
		P[u,v]=pp
	else:
		adj[u]=[(u,v)]
		P[u,v]=pp

	if v in radj.keys():
		radj[v].append((u,v))
		rP[v,u]=pp
	else:
		radj[v]=[(u,v)]
		rP[v,u]=pp

#radj is used for the alpha_ji's
#add empty lists to for nodes that don't have directed edges
for i in list(range(0,numV)):
	if i not in adj.keys():
		adj[i]=[]
	if i not in radj.keys():
		radj[i] = []
print("Graph has been built")

#Discount adoption function assigner
temp=pandas.read_csv(func_csv,sep=",",header=None)

Type={}

for lines in temp[0]:
	line=lines.split()
	if int(line[1])==4: #carryover from previous code. Not sure this is needed
		line[1]=3
	Type[int(line[0])]=int(line[1])

LPSolver = SolverFactory("gurobi")

budgetlist = [10,20,30,40,50]
#budgetlist=[10]

base_dir = output_dir + "/Alpha=1"
#base_dir = "C:/Users/Jeremy Berg/OneDrive - Naval Postgraduate School/01- Student/99 Thesis/Code/CIM-master-Robust/Results/Alpha=1"
seeded_nodes={}
seeded_discounts={}
inf_list={}
obj_vals={}

for budget in budgetlist:
	seeded_nodes[budget]=[]
	seeded_discounts[budget]=[]
	budget_file = open(base_dir + f"/B={budget}.txt")
	found=False
	for line in budget_file:
		splitline = line.split(sep='\t')
		if found:
			seeded_nodes[budget].append(int(splitline[0]))
			seeded_discounts[budget].append(float(splitline[1]))
		if splitline[0]=="CD":
			found=True

discounts={}
for budget in budgetlist:
	for i in list(range(0,numV)):
		if i in seeded_nodes[budget]:
			discounts[budget,i]=seeded_discounts[budget][seeded_nodes[budget].index(i)]
		else:
			discounts[budget,i]=0
#creates a full discount list from imported values
#turn dictionary into 2 dimensional key

for budget in budgetlist:
	print(f"============= Current Budget: {budget} =============")
	if budget == budgetlist[0]:
		pimodel = pyo.ConcreteModel()
		pimodel.pi = pyo.Var(pyo.Set(initialize=list(range(0,numV))),domain=pyo.NonNegativeReals, bounds=(0,1))

		pimodel.con1=pyo.ConstraintList()
		for i,j in list(P.keys()):
			pimodel.con1.add(expr=pimodel.pi[i] - pimodel.pi[j] <= 1 - P[i,j])

		pimodel.con2=pyo.ConstraintList()
		for i in list(range(0,numV)):
			pimodel.con2.add(expr= -pimodel.pi[i]<= -ProbFunc(Type[i],discounts[budget,i]))

		pimodel.obj=pyo.Objective(expr=sum(pimodel.pi[i] for i in list(range(0,numV))), sense=pyo.minimize)
		LPSolver.solve(pimodel,tee=False)
		fullpi=pyo.value(pimodel.obj)
	else:
		for i in list(range(0,numV)):
			pimodel.con2[i+1].set_value(expr= -pimodel.pi[i]<= -ProbFunc(Type[i],discounts[budget,i]))
		LPSolver.solve(pimodel, tee=False)
		fullpi=pyo.value(pimodel.obj)

	print(f"Budget {budget} full pi: {fullpi}")


import datetime
import itertools
import json
import os
from functools import cache
from pathlib import Path

from connexion import FlaskApp
from connexion.resolver import RelativeResolver
from dotenv import load_dotenv
from flask import request
from flask_cors import CORS
from pony.orm import Database, set_sql_debug, db_session

from src.models import define_entities_game, define_entities_meta, define_entities_user, GameTypeEnum, GameLengthEnum, \
    GroupSizeEnum, GroupNeedEnum, define_entities_stats, define_entities_api
from src.services.enforcedefaults import validator_remap

# Load project root and constants from dotenv file
ROOT = Path(__file__).parent.parent
dotenv_path = ROOT.joinpath('.env')
load_dotenv(dotenv_path)

@cache
def get_db(users_db=False):
    """Bind Pony ORM to database
    This acts like a singleton because of caching.
    """
    if users_db:
        credentials = json.loads(os.environ.get("DB_USERS_CONNECT"))
    else:
        credentials = json.loads(os.environ.get("DB_CONNECT"))

    if not os.environ.get("FLASK_DEBUG"):  # pragma: no cover / production
        database = Database(provider='mysql', **credentials)
    elif os.environ.get('FLASK_TESTING'):
        database = Database(provider='sqlite', filename=f':memory:', create_db=True)
    else:  # pragma: no cover / development
        database = Database(provider='sqlite', filename=f'{credentials["db"]}.sqlite', create_db=True)

    if users_db:
        define_entities_user(database)
    else:
        define_entities_game(database)
        define_entities_meta(database)
        define_entities_stats(database)
        define_entities_api(database)

    database.generate_mapping(create_tables=True)
    set_sql_debug(False)
    return database

@cache
def get_app():
    """Connexion Wrapper App
    Connexion creates a wrapper around a Flask app, through which all requests are passed.
    The Connexion wrapper app is called by wsgi.py and pytest.
    You can access the Flask app with `app.app.properties`.
    """

    # Todo: Sentry integration
    # sentry_sdk.init(
    #     dsn=os.environ.get('SENTRY_DSN'),
    #     integrations=[FlaskIntegration()],
    #     # Reduce this for production, maybe 0.25
    #     traces_sample_rate=0.25,
    #     release=get_project_version(),
    #     environment=Config().SENTRY
    # )

    swagger_options = {
        'swagger_url': '/',
        'swagger_ui_config': {
            'operationsSorter': 'method',
            'tagsSorter': 'alpha',
        }
    }
    connexion_app = FlaskApp(__name__, specification_dir=ROOT, options=swagger_options)
    CORS(connexion_app.app)
    connexion_app.add_api(
        'openapi.yml',
        resolver=RelativeResolver('src.views'),
        validate_responses=True,
        validator_map=validator_remap,
    )
    connexion_app.app.config.from_prefixed_env()

    @connexion_app.app.after_request
    @db_session
    def activity_tracking(response):
        from src.services import tracking
        if response.status_code in [200, 201]:
            session = tracking.get_session(request)
            tracking.add_request(session, request, response)
        return response

    return connexion_app

@db_session
def run_startup_tasks(db):  # pragma: no cover
    if hasattr(db, 'User'):
        if db.User.get(login="admin"):
            return
        startup_users_db(db)
    else:
        if db.APIInfo.get(name="aleleio-api"):
            return
        startup_api_info(db)
        startup_games_db(db)

def startup_users_db(udb):  # pragma: no cover
    for user in [os.environ.get("USER_ADMIN"), os.environ.get("USER_WEB"), os.environ.get("USER_ANDROID"), os.environ.get("USER_IOS")]:
        udb.User(**json.loads(user))

def startup_games_db(db):
    """Seed database with category enums if not already present
    """
    for item in GameTypeEnum:
        db.GameType(slug=item.value, full=item.full)
    for item in GameLengthEnum:
        db.GameLength(slug=item.value, full=item.full)
    for item in GroupSizeEnum:
        db.GroupSize(slug=item.value, full=item.full)
    for item in GroupNeedEnum:
        db.GroupNeed(slug=item.value, full=item.full)

    for item in itertools.chain(GameTypeEnum, GameLengthEnum, GroupSizeEnum, GroupNeedEnum):
        db.QueryParam(slug=item.value)

def startup_api_info(db):
    from src.services.api_info import get_project_version

    for project in ["aleleio-api", "aleleio-web", "teambuilding-games"]:
        params = dict(
            name=project,
            version=get_project_version(project),
            last_commit=datetime.datetime.utcnow(),
            url=f"https://github.com/aleleio/{project}"
        )
        db.APIInfo(**params)


from collections import UserList

from metadata.models import DataSource
from metadata.models.constants import IGNORED_CONSUL_SYNC_DATA_IDS
from metadata.models.influxdb_cluster import InfluxDBClusterInfo

class MockHashConsul(object):
    result_list = []

    def put(self, key, value):
        self.result_list.append({"key": key, "value": value})

class MockClusterList(UserList):
    def count(self):
        return len(self.data)

class TestConsulCluster(object):
    def test_refresh_consul_cluster_config(self, mocker, patch_redis_tools):
        cluster_host_list = MockClusterList()
        cluster_host_list.append(InfluxDBClusterInfo(host_name="host1", cluster_name="cluster1", host_readable=True))
        cluster_host_list.append(InfluxDBClusterInfo(host_name="host2", cluster_name="cluster1", host_readable=False))
        mocker.patch("metadata.models.influxdb_cluster.InfluxDBClusterInfo.objects.all", return_value=cluster_host_list)
        mock_hash_consul = MockHashConsul()
        mocker.patch("metadata.utils.consul_tools.HashConsul", return_value=mock_hash_consul)
        InfluxDBClusterInfo.refresh_consul_cluster_config()
        assert mock_hash_consul.result_list[0]["key"] == "/".join([InfluxDBClusterInfo.CONSUL_PREFIX_PATH, "cluster1"])
        assert mock_hash_consul.result_list[0]["value"] == {
            "host_list": ["host1", "host2"],
            "unreadable_host_list": ["host2"],
        }

    def test_refresh_consul_data_source_config(self, mocker, patch_redis_tools):

        data_source_list = [DataSource(bk_data_id=data_id) for data_id in IGNORED_CONSUL_SYNC_DATA_IDS]
        data_source_list.append(DataSource(bk_data_id=12345))
        mocker.patch("metadata.models.DataSource.objects.all", return_value=data_source_list)
        mocker.patch("metadata.models.DataSource.to_json", return_value={})
        mock_hash_consul = MockHashConsul()
        mocker.patch("metadata.utils.consul_tools.HashConsul", return_value=mock_hash_consul)
        for ds in data_source_list:
            ds.refresh_consul_config()

        assert len(mock_hash_consul.result_list) == 1
        assert mock_hash_consul.result_list[0]["key"] == DataSource(bk_data_id=12345).consul_config_path


#!/usr/bin/env python
# coding: utf-8

# # Data Preprocessing

# ### Importing the libraries

# In[ ]:

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# ### Reading the dataset

# In[ ]:

dataset = pd.read_csv('startups.csv')
dataset

# In[ ]:

dataset.describe()

# In[ ]:

# Separate Independent variables (X) and dependent variable (y)

X = dataset.iloc[:, 0:4].values
y = dataset.iloc[:, 4].values

print(X,'\n\n', y)

# ### Missing values

# In[ ]:

# Replace missing values by the column/variable average (mean)
# Import Class --> Create Object --> Fit Object to Data --> Transform Data

from sklearn.impute import SimpleImputer                            #import the SimpleImputer class
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')     #create the imputer object

imputer.fit(X[:, 0:3])                          #fit the object to the data
X[:, 0:3] = imputer.transform(X[:, 0:3])        #transform data

print(X)

# ### Encoding categorical variables

# In[ ]:

# Encoding independent variables
# Import Class --> Create Object --> Fit Object to Data --> Transform Data

from sklearn.compose import ColumnTransformer       # import class
from sklearn.preprocessing import OneHotEncoder     # import class

ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(drop = 'first'), [3])], remainder='passthrough')
#create object

X = np.array(ct.fit_transform(X))                   # fit object to data and transform data

print(X)

# ### Feature scaling

# In[ ]:

# Adjusting the scales of independent variables
# Import Class --> Create Object --> Fit Object to Data --> Transform Data

from sklearn.preprocessing import StandardScaler   # import class
sc = StandardScaler()                              # create object
X = sc.fit_transform(X)                            # fit and transform

print(X)

# In[ ]:

df = pd.DataFrame(X)
df.describe()

# ### Splitting the dataset into the Training set and Test set

# In[ ]:

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)

# In[ ]:

print(X_train)

# # Artificial Neural Net (ANN) for Regression

# ### Training the Model

# In[ ]:

# We use ANNs.
# Can also use linear, polynomial, support vector, decision trees or random forest regression.

# Import Class --> Create Object --> Fit Object to Data --> Predict

import tensorflow as tf               # import class
from tensorflow import keras

regressor = keras.Sequential([
    tf.keras.layers.Dense(units = 20, input_dim=5, activation='relu'),
    tf.keras.layers.Dense(units = 20, activation='relu'),
    tf.keras.layers.Dense(units = 20, activation='relu'),
    tf.keras.layers.Dense(units = 10, activation='relu'),
    tf.keras.layers.Dense(units = 1, activation = 'linear')
])                                                         # create object with the required layers

regressor.compile(
              loss='mean_squared_error',
              optimizer=tf.keras.optimizers.RMSprop(0.01),
              batch_size=20,
              metrics=['mean_absolute_error']
)                                                          # compile object selecting options

regressor.fit(X_train, y_train, epochs= 500)               # fit object and decide the number of epochs

# ### Testing the Model

# In[ ]:

# make predictions for test data

y_pred = regressor.predict(X_test).flatten()    # predict (flatten() reshapes y_pred to a one-dimensional array)

err = abs(y_test - y_pred)
df = pd.DataFrame({'y_pred': y_pred, 'y_test': y_test, 'error': err})     # compare
df

# In[ ]:

# determine model accuracy on the test set

loss, mae = regressor.evaluate(X_test, y_test, verbose=0)
print('Mean Absolute Error =', mae)

# ### Graphing (Optional)

# In[ ]:

plt.bar(range(10), err)

# In[ ]:

plt.bar(range(10), y_test)

# ### Forecasting (Optional)

# In[ ]:

# What is the profit for
# a California startup spending 10,000 on R&D, 50,000 on Administration and 10,000 on Marketing
# What happens if they spend 100,000 on R&D?

Z = sc.transform([[0, 0, 10000, 50000, 10000]])     # scale data using .transform() method but NOT .fit_transform()

prediction = regressor.predict(Z)
print('Scaled features = ', Z, ', Forecast = ', prediction)

# In[ ]:


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Feb 23 15:31:46 2018
script python to plot some outputs of the model
@author: jbrlod
modified by mkouassi & arimoux
"""
import os
import numpy as np
import matplotlib.pyplot as plt
import shutil
import xarray as xr
from baseutil import dataset, weights_mask

alltestname = ['BaseTest_Cloud1.nc','BaseTest_Clouds.nc','BaseTest_Square1.nc','BaseTest_Squares.nc']
#alltestname = ['base_validation_1cloud.nc', 'base_validation_multiple_clouds.nc', 'base_validation_1square.nc','base_validation_multiple_squares.nc']
## DATA DIRECTORY
AllInputDs = ["cl1", "cl+", "sq1", "sq+"]              # cl1 : cloud ; sq1: nauge carré ; sq2 : nuage carré centré
allPattern = ['cloud1','clouds','square1','squares'];  # list of the cloud's patterns
alldir = ['cloud1','clouds','square1','squares'];      # list of all data directories
AllDeepModels = ['cnn','gan']                          # lists of deep learning algorithms
deepmodels=AllDeepModels[0]
count = 0;
for icode, inputds in enumerate(AllInputDs):
    if (inputds==AllInputDs[0]):
        ddir = alldir[0]
        datadir = '../data/cloud/'+ddir
        pattern = allPattern[0]
        testname = alltestname[0]
        weightsName = 'Weights_Cloud1.nc'

    elif (inputds == AllInputDs[1]):
        ddir = alldir[1]
        datadir = '../data/cloud/'+ddir
        pattern = allPattern[1]
        testname = alltestname[1]
        weightsName = 'Weights_Clouds.nc'

    elif (inputds == AllInputDs[2]):
        ddir = alldir[2]
        datadir = '../data/square/'+ddir
        pattern = allPattern[2]
        testname = alltestname[2]
        weightsName = 'Weights_Square1.nc'

    elif (inputds == AllInputDs[3]):
        ddir = alldir[3]
        datadir = '../data/square/'+ddir
        pattern = allPattern[3]
        testname = alltestname[3]
        weightsName = 'Weights_Squares.nc'

    # CALL THE DATASET CLASS
    ds = dataset(basename=os.path.join(datadir,testname))

    # BUILDING of Binary input mask  and loss function weight masks
    Inputds = os.path.join(datadir,testname)
    weightdsName = os.path.join(datadir,weightsName)
    wds = weights_mask(Inputds, weightdsName, weight_c=0.1, weight_n=1)
    weight = wds.weights.expand_dims('canal',3)
    Xbinary = wds.bmask.expand_dims('canal',3)
    ## Concatenation des bases des dataArray 'yt',weights
    yt_2dim = xr.concat((ds.yt, weight),dim='canal')

    ## MODEL DIRECTORY
    from keras.models import load_model
    from modelutil import context_mse, masked_mse

    allLoss = ['mmse','cmse'];
    allLayers=['2layers','3layers','4layers']; nbLayers = allLayers[2]
    # list of models directories based on the inputs data models
    allmdir=['classic','binary']#, 'temporal' ,'bintemp']
    for mcode, mdir in enumerate(allmdir):
        modelsdir = '../data/models/'+deepmodels+'/'+mdir
        #
        if (mdir == allmdir[0]):
            Xbinary.values = np.array(Xbinary.values, dtype='int')
        elif (mdir == allmdir[1]):
            Xbinary.values = np.ones_like(Xbinary.values)

        X_2dim = xr.concat((ds.X, Xbinary),dim='canal')
        ## Splitting
        #X_test, X_valid, yt_test, yt_valid = train_test_split(X_2dim, yt_2dim, test_size=0.2)

        # Choice of the loss function
        for mcode, Lossf in enumerate(allLoss):
            # Name of the neural architecture
            name = 'model_'+nbLayers+'_'+pattern+'_'+Lossf+'_'+mdir
            # Outputname
            outname = 'DatasetNN_'+nbLayers+'_'+pattern+'_'+Lossf+'_'+mdir+'_'+deepmodels+'.nc'
            # Model loading
            if Lossf == allLoss[0]:
                model = load_model(os.path.join(modelsdir,name), custom_objects={'masked_mse':masked_mse})
            elif Lossf == allLoss[1]:
                model = load_model(os.path.join(modelsdir,name),custom_objects={'context_mse':context_mse})
            else:
                raise ValueError("Choisir une fonction de cout existante ou en créer une dans modelutil")

            ypredict = xr.DataArray(model.predict(X_2dim,batch_size=10,verbose=1),coords=ds.yt.coords)

            # Generate a combinated image of original & predicted images
            isCloud = np.equal(ds.X,0)
            yfinal = (1-isCloud)*ds.X + isCloud*ypredict

            # Save prediction
            dsout = xr.Dataset({'X':ds.X,'yt':ds.yt,'ypredict':ypredict,'yfinal':yfinal,'ytfull':ds.ytfull})
            dsout.to_netcdf(os.path.join(datadir,outname))

            print(count); count +=1

            # Plot some random images
            PLOT = False

            #save the images
            SAVE = False

            if PLOT:

                nim = 20 #number of images to be plot
                ii = np.random.randint(0,dsout.index.size,nim)
                vmin, vmax = -1.5,0

                for i,ind in enumerate(ii):
                    fig, axes= plt.subplots(ncols=4)
                    axes[0].imshow(np.log10(dsout.X[ind].squeeze()),vmin=vmin,vmax=vmax)
                    axes[1].imshow(np.log10(dsout.yt[ind].squeeze()),vmin=vmin,vmax=vmax)
                    axes[2].imshow(np.log10(dsout.ypredict[ind].squeeze()),vmin=vmin,vmax=vmax)
                    axes[3].imshow(np.log10(dsout.yfinal[ind].squeeze()),vmin=vmin,vmax=vmax)
                    title = 'Image_' + str(int(dsout.index[ind]))
                    plt.suptitle(title)
                    if SAVE:
                        # example dir : A FAIRE +++
                        exampledir = os.path.join('../figures/examples',os.path.splitext(name)[0])
                        if os.path.isdir():
                            print('Directory already created')
                            plt.savefig(os.path.join(exampledir,title+'.png'))
                        else:
                            shutil.rmtree(exampledir,ignore_errors=True)
                            os.makedirs(exampledir)
                            plt.savefig(os.path.join(exampledir,title+'.png'))


from fastapi import APIRouter, Depends, HTTPException, status
import typing as t
from app.elastic.session import get_cluster_by_user, get_cluster_by_key
from app.elastic.schemas import (User, Token, Authenticated, ApiKey,
                                Project, ProjectCreate, ProjectUpdate, Mapping,
                                Query, Annotation, Annotations)

mainrouter = r = APIRouter()

@r.post("/token", response_model=Token)
async def login(user: User, es=Depends(get_cluster_by_user))-> dict:
    try:
        token = es.security.create_api_key(body={'name':user.username, 'expiration':'1d'})
        api_key = token.get('api_key')
        api_key_id = token.get('id')
        return {"id": api_key_id, "api_key": api_key}
    except:
        raise HTTPException(
            status_code=status.HTTP_401_UNAUTHORIZED,
            detail="Incorrect email or password"
        )

@r.get("/authenticate", response_model=Authenticated)
async def authenticate(es=Depends(get_cluster_by_key))->dict:
    try:
        resp = es.security.authenticate()
        return {'is_authenticated': True}
    except:
        return {'is_authenticated': False}


from pathlib import Path
import textwrap

def delete_temp_files(path: str = "temp") -> None:
    folder = Path(path)
    for file in folder.glob("*"):
        if file.is_file() and file.name != ".gitignore":
            try:
                file.unlink()
            except BaseException as e:
                print(f"File '{file}', can not delete. \n{e}")

def split_text(text: str, max_length: int = 4000) -> list[str]:
    wrapped_text = textwrap.wrap(text=text, width=max_length, break_long_words=False)
    return wrapped_text


# This class is to create a stream for the required song or download it

# This youtube_dl is used to search the songs and download it from the youtube platform
import youtube_dl

# This Embed class is used to create a custom embed message for the specific class
import discord

# This is used to use the bot functions and commands
from discord.ext import commands

# This is used to implement the asynchronous operations like building a dynamic queue
import asyncio

# This is used to create a coroutine in the async event loop
from functools import partial

# This is used to create a asynchronous timer
from async_timeout import timeout

YTDL_OPTS = {
    "default_search": "auto",
    "format": "bestaudio/best",
    "quiet": True,
    "noplaylist": True,
    "restrictfilenames": True,
    "no_warnings": True,
    "outtmpl": 'downloads/%(extractor)s-%(id)s-%(title)s.%(ext)s',
    'nocheckcertificate': True,
    'ignoreerrors': False,
    'logtostderr': False,
    'source_address': '0.0.0.0'  # ipv6 addresses cause issues sometimes
}

ffmpegopts = {
    'before_options': '-nostdin -reconnect 1 -reconnect_streamed 1 -reconnect_delay_max 5',
    'options': '-vn'
}

ytdl = youtube_dl.YoutubeDL(YTDL_OPTS)

class YTDLSource(discord.PCMVolumeTransformer):

    def __init__(self, source, *, data, requester):
        super().__init__(source)

        self.requester = requester

        self.title = data.get('title')
        self.web_url = data.get('webpage_url')

    def __getitem__(self, item: str):
        # This funtion allows us to access attributes similar to a  dictionary accessing
        return self.__getattribute__(item)

    @classmethod
    async def create_source(cls, ctx, search: str, *, loop, download=False):
        loop = loop or asyncio.get_event_loop()

        to_run = partial(ytdl.extract_info, url=search, download=download)
        data = await loop.run_in_executor(None, to_run)

        if 'entries' in data:
            data = data['entries'][0]

        if download:
            source = ytdl.prepare_filename(data)
        else:
            return {'webpage_url': data['webpage_url'], 'requester': ctx.author, 'title': data['title']}

        return cls(discord.FFmpegPCMAudio(source, **ffmpegopts), data=data, requester=ctx.author)

    @classmethod
    async def regather_stream(cls, data, *, loop):
        # It is used to prepare a stream, instead of downloading as the youtube links will expire.
        loop = loop or asyncio.get_event_loop()
        requester = data['requester']

        to_run = partial(ytdl.extract_info, url=data['webpage_url'], download=False)
        data = await loop.run_in_executor(None, to_run)

        return cls(discord.FFmpegPCMAudio(source=data['url'], **ffmpegopts), data=data, requester=requester)


from pathlib import Path

import pytest

from flood_adapt.object_model.direct_impact.socio_economic_change import (
    SocioEconomicChange,
)
from flood_adapt.object_model.hazard.physical_projection import (
    PhysicalProjection,
)
from flood_adapt.object_model.projection import Projection

test_database = Path().absolute() / "tests" / "test_database"

def test_projection_read(cleanup_database):
    test_toml = (
        test_database
        / "charleston"
        / "input"
        / "projections"
        / "all_projections"
        / "all_projections.toml"
    )
    assert test_toml.is_file()

    projection = Projection.load_file(test_toml)

    # Assert that the configured risk drivers are set to the values from the toml file
    assert isinstance(projection.get_physical_projection(), PhysicalProjection)
    assert isinstance(projection.get_socio_economic_change(), SocioEconomicChange)
    assert projection.attrs.name == "all_projections"
    assert projection.attrs.description == "all_projections"
    assert projection.get_physical_projection().attrs.sea_level_rise.value == 2
    assert projection.get_socio_economic_change().attrs.economic_growth == 20
    with pytest.raises(AttributeError):
        projection.get_socio_economic_change().attrs.sea_level_rise.value
    with pytest.raises(AttributeError):
        projection.get_physical_projection().attrs.economic_growth

def test_projection_only_slr(cleanup_database):
    test_toml = (
        test_database
        / "charleston"
        / "input"
        / "projections"
        / "SLR_2ft"
        / "SLR_2ft.toml"
    )

    assert test_toml.is_file()

    projection = Projection.load_file(test_toml)

    # Assert that all unconfigured risk drivers are set to the default values
    assert projection.get_physical_projection().attrs.storm_frequency_increase == 0

    # Assert that the configured risk drivers are set to the values from the toml file
    assert projection.attrs.name == "SLR_2ft"
    assert projection.attrs.description == "SLR_2ft"
    assert projection.get_physical_projection().attrs.sea_level_rise.value == 2
    assert projection.get_physical_projection().attrs.sea_level_rise.units == "feet"


from django.conf.urls import include, url
from django.contrib import admin
from rest_framework.authtoken import views

urlpatterns = [
    #Apps
    url(r'^api/v1/amazon/', include('amazon.urls')),
    url(r'^api/v1/tripadvisor/', include('tripadvisor.urls')),
    url(r'^api/v1/facebook/', include('facebook.urls')),
    url(r'^api/v1/instagram/', include('instagram.urls')),
    url(r'^api/v1/youtube/', include('youtube.urls')),
    url(r'^api/v1/lotame/', include('lotame.urls')),
    url(r'^api/v1/promodescuentos/', include('promodescuentos.urls')),

    # Token
    url(r'^admin/', admin.site.urls),
    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    url(r'api-token-auth/', views.obtain_auth_token, name='api-token-auth'),
]