from typing import Optional
from decimal import Decimal

from validator_collection import validators

from highcharts_core import errors
from highcharts_core.metaclasses import HighchartsMeta
from highcharts_core.decorators import class_sensitive
from highcharts_core.options.sonification.track_configurations import (InstrumentTrackConfiguration,
                                                                       SpeechTrackConfiguration,
                                                                       ContextTrackConfiguration)
from highcharts_core.options.sonification.grouping import SonificationGrouping
from highcharts_core.utility_classes.events import SonificationEvents

class SonificationOptions(HighchartsMeta):
    """Options for configuring sonification and audio charts."""

    def __init__(self, **kwargs):
        self._after_series_wait = None
        self._default_instrument_options = None
        self._default_speech_options = None
        self._duration = None
        self._enabled = None
        self._events = None
        self._global_context_tracks = None
        self._global_tracks = None
        self._master_volume = None
        self._order = None
        self._point_grouping = None
        self._show_crosshair = None
        self._show_tooltip = None
        self._update_interval = None

        self.after_series_wait = kwargs.get('after_series_wait', None)
        self.default_instrument_options = kwargs.get('default_instrument_options', None)
        self.default_speech_options = kwargs.get('default_speech_options', None)
        self.duration = kwargs.get('duration', None)
        self.enabled = kwargs.get('enabled', None)
        self.events = kwargs.get('events', None)
        self.global_context_tracks = kwargs.get('global_context_tracks', None)
        self.global_tracks = kwargs.get('global_tracks', None)
        self.master_volume = kwargs.get('master_volume', None)
        self.order = kwargs.get('order', None)
        self.point_grouping = kwargs.get('point_grouping', None)
        self.show_crosshair = kwargs.get('show_crosshair', None)
        self.show_tooltip = kwargs.get('show_tooltip', None)
        self.update_interval = kwargs.get('update_interval', None)

    @property
    def after_series_wait(self) -> Optional[int | float | Decimal]:
        """The time to wait in milliseconds after each data series when playing the visualization's data series
        in sequence. Defaults to ``700``.

        :rtype: numeric or :obj:`None <python:None>`
        """
        return self._after_series_wait

    @after_series_wait.setter
    def after_series_wait(self, value):
        self._after_series_wait = validators.numeric(value,
                                                     allow_empty = True,
                                                     minimum = 0)

    @property
    def default_instrument_options(self) -> Optional[InstrumentTrackConfiguration]:
        """Default sonification options for all instrument tracks.

        .. warning::

          If specific options are also set on individual tracks or per-series, this configuration will be *overridden*.

        :rtype: :class:`InstrumentTrackConfiguration <highcharts_core.options.sonification.track_configurations.InstrumentTrackConfiguration>`
          or :obj:`None <python:None>`
        """
        return self._default_instrument_options

    @default_instrument_options.setter
    @class_sensitive(InstrumentTrackConfiguration)
    def default_instrument_options(self, value):
        self._default_instrument_options = value

    @property
    def default_speech_options(self) -> Optional[SpeechTrackConfiguration]:
        """Default sonification options for all speech tracks.
        .. warning::

          If specific options are also set on individual tracks or per-series, this configuration will be *overridden*.

        :rtype: :class:`SpeechTrackConfiguration <highcharts_core.options.sonification.track_configurations.SpeechTrackConfiguration>`
          or :obj:`None <python:None>`
        """
        return self._default_speech_options

    @default_speech_options.setter
    @class_sensitive(SpeechTrackConfiguration)
    def default_speech_options(self, value):
        self._default_speech_options = value

    @property
    def duration(self) -> Optional[int | float | Decimal]:
        """The total duration of the sonification, expressed in milliseconds. Defaults to ``6000``.

        :rtype: numeric or :obj:`None <python:None>`
        """
        return self._duration

    @duration.setter
    def duration(self, value):
        self._duration = validators.numeric(value, allow_empty = True, minimum = 0)

    @property
    def enabled(self) -> Optional[bool]:
        """If ``True``, enables sonification functionality on the chart. Defaults to ``True``.

        :rtype: :class:`bool <python:bool>` or :obj:`None <python:None>`
        """
        return self._enabled

    @enabled.setter
    def enabled(self, value):
        if value is None:
            self._enabled = None
        else:
            self._enabled = bool(value)

    @property
    def events(self) -> Optional[SonificationEvents]:
        """Event handlers for sonification.

        :rtype: :class:`SonificationEvents <highcharts_core.utility_classes.events.SonificationEvents>` or
          :obj:`None <python:None>`
        """
        return self._events

    @events.setter
    @class_sensitive(SonificationEvents)
    def events(self, value):
        self._events = value

    @property
    def global_context_tracks(self) -> Optional[ContextTrackConfiguration]:
        """Context tracks to add globally, an array of either instrument tracks, speech tracks, or a mix.

        .. note::

          Context tracks are not tied to data points, but play at a set interval - either based on ``time`` or on
          ``prop`` values.

        :rtype: :class:`ContextTrackConfiguration <highcharts_core.options.sonification.track_configurations.ContextTrackConfiguration>`
          or :obj:`None <python:None>`
        """
        return self._global_context_tracks

    @global_context_tracks.setter
    @class_sensitive(ContextTrackConfiguration)
    def global_context_tracks(self, value):
        self._global_context_tracks = value

    @property
    def global_tracks(self) -> Optional[InstrumentTrackConfiguration]:
        """Global tracks to add to every series.

        :rtype: :class:`InstrumentTrackConfiguration <highcharts_core.options.sonification.track_configurations.InstrumentTrackConfiguration>`
          or :obj:`None <python:None>`
        """
        return self._global_tracks

    @global_tracks.setter
    @class_sensitive(InstrumentTrackConfiguration)
    def global_tracks(self, value):
        self._global_tracks = value

    @property
    def master_volume(self) -> Optional[int | float | Decimal]:
        """The overall/master volume for the sonification, from ``0`` to ``1``. Defaults to ``0.7``.

        :rtype: numeric or :obj:`None <python:None>`
        """
        return self._master_volume

    @master_volume.setter
    def master_volume(self, value):
        self._master_volume = validators.numeric(value,
                                                 allow_empty = True,
                                                 minimum = 0,
                                                 maximum = 1)

    @property
    def order(self) -> Optional[str]:
        """The order in which to play the sonification for data series. Accepts either:

          * ``'sequential'`` where the series play individually one after the other or
          * ``'simultaneous'`` where the series play at the same time

        Defaults to ``'sequential'``.

        :rtype: :class:`str <python:str>` or :obj:`None <python:None>`
        """
        return self._order

    @order.setter
    def order(self, value):
        if not value:
            self._order = None
        else:
            value = validators.string(value)
            value = value.lower()
            if value not in ['sequential', 'simultaneous']:
                raise errors.HighchartsValueError(f'.order expects either "sequential" or "simultaenous". '
                                                  f'Received: "{value}"')

            self._order = value

    @property
    def point_grouping(self) -> Optional[SonificationGrouping]:
        """Options for grouping data points together when sonifying.

        This allows for the visual presentation to contain more points than what is being played.

        If not enabled, all visible / uncropped points are played.

        :rtype: :class:`SonificationGrouping <highcharts_core.options.sonification.grouping.SonificationGrouping>` or
          :obj:`None <python:None>`
        """
        return self._point_grouping

    @point_grouping.setter
    @class_sensitive(SonificationGrouping)
    def point_grouping(self, value):
        self._point_grouping = value

    @property
    def show_crosshair(self) -> Optional[bool]:
        """If ``True``, show X and Y crosshairs (if defined on the chart) as the sonification plays. Defaults to
        ``True``.

        .. warning::

           If multiple tracks that play at different times try to show crosshairs, it can be glitchy. Therefore,
           it is recommended in those cases to turn this on/off for individual tracks using the ``.show_play_marker``
           property.

        :rtype: :class:`bool <python:bool>` or :obj:`None <python:None>`
        """
        return self._show_crosshair

    @show_crosshair.setter
    def show_crosshair(self, value):
        if value is None:
            self._show_crosshair = None
        else:
            self._show_crosshair = bool(value)

    @property
    def show_tooltip(self) -> Optional[bool]:
        """If ``True``, show tooltips as the sonification plays. Defaults to ``True``.

        .. warning::

           If multiple tracks that play at different times try to show tooltips, it can be glitchy. Therefore,
           it is recommended in those cases to turn this on/off for individual tracks using the ``.show_play_marker``
           property.

        :rtype: :class:`bool <python:bool>` or :obj:`None <python:None>`
        """
        return self._show_tooltip

    @show_tooltip.setter
    def show_tooltip(self, value):
        if value is None:
            self._show_tooltip = None
        else:
            self._show_tooltip = bool(value)

    @property
    def update_interval(self) -> Optional[int | float | Decimal]:
        """The number of milliseconds to wait between each recomputation of the sonification, if the chart updates
        rapidly.

        .. tip::

          This avoids slowing down processes like panning.

        :rtype: numeric or :obj:`None <python:None>`
        """
        return self._update_interval

    @update_interval.setter
    def update_interval(self, value):
        self._update_interval = validators.numeric(value, allow_empty = True, minimum = 0)

    @classmethod
    def _get_kwargs_from_dict(cls, as_dict):
        kwargs = {
            'after_series_wait': as_dict.get('afterSeriesWait', None),
            'default_instrument_options': as_dict.get('defaultInstrumentOptions', None),
            'default_speech_options': as_dict.get('defaultSpeechOptions', None),
            'duration': as_dict.get('duration', None),
            'enabled': as_dict.get('enabled', None),
            'events': as_dict.get('events', None),
            'global_context_tracks': as_dict.get('globalContextTracks', None),
            'global_tracks': as_dict.get('globalTracks', None),
            'master_volume': as_dict.get('masterVolume', None),
            'order': as_dict.get('order', None),
            'point_grouping': as_dict.get('pointGrouping', None),
            'show_crosshair': as_dict.get('showCrosshair', None),
            'show_tooltip': as_dict.get('showTooltip', None),
            'update_interval': as_dict.get('updateInterval', None),
        }

        return kwargs

    def _to_untrimmed_dict(self, in_cls = None) -> dict:
        untrimmed = {
            'afterSeriesWait': self.after_series_wait,
            'defaultInstrumentOptions': self.default_instrument_options,
            'defaultSpeechOptions': self.default_speech_options,
            'duration': self.duration,
            'enabled': self.enabled,
            'events': self.events,
            'globalContextTracks': self.global_context_tracks,
            'globalTracks': self.global_tracks,
            'masterVolume': self.master_volume,
            'order': self.order,
            'pointGrouping': self.point_grouping,
            'showCrosshair': self.show_crosshair,
            'showTooltip': self.show_tooltip,
            'updateInterval': self.update_interval,
        }

        return untrimmed


import torch
from torch.utils.data import DataLoader
import torchaudio
import time
import warnings
warnings.filterwarnings('ignore')

yesno_data = torchaudio.datasets.YESNO('.', download=False)

def collate_fn(batch):

    tensors = [b[0].t() for b in batch if b]
    tensors = torch.nn.utils.rnn.pad_sequence(tensors, batch_first=True)
    tensors = tensors.transpose(1, -1)

    targets = torch.tensor([b[2] for b in batch if b])
    targets = torch.nn.utils.rnn.pad_sequence(targets, batch_first=True)

    return tensors, targets

pin_memory = False
print('pin_memory is', pin_memory)
for num_workers in range(0, 32, 1):
    data_loader = torch.utils.data.DataLoader(
                                                yesno_data,
                                                batch_size=32,
                                                pin_memory=pin_memory,
                                                num_workers=num_workers,
                                                collate_fn=collate_fn)
    start = time.time()
    for epoch in range(1, 5):
        for i, (data, _) in enumerate(data_loader):
            torch.tensor(data).to('cuda')
    end = time.time()
    print("Finish with:{} second, num_workers={}".format(end - start, num_workers))

pin_memory = True
print('pin_memory is', pin_memory)
for num_workers in range(0, 32, 1):
    data_loader = torch.utils.data.DataLoader(
                                                yesno_data,
                                                batch_size=32,
                                                pin_memory=pin_memory,
                                                num_workers=num_workers,
                                                collate_fn=collate_fn)
    start = time.time()
    for epoch in range(1, 5):
        for i, (data, _) in enumerate(data_loader):
            torch.tensor(data).to('cuda')
    end = time.time()
    print("Finish with:{} second, num_workers={}".format(end - start, num_workers))


# -*- coding: utf-8 -*-
"""
Created on Sun Jun 25 20:12:52 2017

@author: dell
"""
import collections
def topKFrequent(nums, k):
        """
        :type nums: List[int]
        :type k: int
        :rtype: List[int]
        """
        cnt=collections.Counter(nums)
        sort_fre=sorted(cnt.items(),key=lambda d:d[1],reverse=True)
        alpha=sort_fre[:k]
        return [x[0] for x in alpha]

s=[1,1,1,2,2,3,3]
print(topKFrequent(s, 2))


import identifier_phase, biomarker, clinical_trial_code, patient_number, lines_of_therapy, study_evaluation
import utils.chunk_utils as cu
import utils.json_utils as ju

def get_overall_info(content):
    overall_dict = {}
    identifier_phase_dicts = identifier_phase.get_final_result(content)
    overall_dict['Identifier'] = ','.join([d['Identifier'] for d in identifier_phase_dicts])
    overall_dict['Phase'] = ','.join([d['Phase'] for d in identifier_phase_dicts])
    overall_dict['Drug'] = ','.join([d['Drug'] for d in identifier_phase_dicts])
    clinical_trial_code_dicts = clinical_trial_code.get_final_result(content)
    overall_dict['Clinical Trial Code'] = ','.join([d['Clinical Trial Code'] for d in clinical_trial_code_dicts])
    biomarker_dicts = biomarker.get_final_result(content)
    overall_dict['Biomarker'] = ','.join([d['Biomarker'] for d in biomarker_dicts])
    # target_indication_dicts = target_indication.get_final_result(content)
    overall_dict['Target Indication'] = ','.join([d['Target Indication'] for d in biomarker_dicts])
    patient_number_dicts = patient_number.get_final_result(content)
    overall_dict['Overall Enrollment'] = ','.join([d['Overall Enrollment'] for d in patient_number_dicts])
    lines_of_therapy_dicts = lines_of_therapy.get_final_result(content)
    overall_dict['Line of Therapy'] = ','.join([d['Line of Therapy'] for d in lines_of_therapy_dicts])
    study_evaluation_dicts = study_evaluation.get_final_result(content)
    overall_dict['Overall Evaluation'] = ','.join([d['Evaluation'] for d in study_evaluation_dicts])
    print()
    print("============================   Overall Result  =============================")
    return overall_dict

if __name__ == '__main__':
    print(ju.json_beautify(get_overall_info(cu.get_file_content("data/tmp_test_data_1"))))


from pyrogram import filters,Client
import wikipedia
wikipedia.set_lang("it")
cprefix="/"

@Client.on_message(filters.command("wiki",cprefix))
async def my_handler(client, message):
    send = await message.reply_text("Cercando...")
    try:
        await send.edit_text(str(wikipedia.summary(message.text.split(" ",1)[1])))
    except Exception as e:
        await send.edit_text(e)
@Client.on_message(filters.command("wikilang",cprefix))
async def lingua(client, message):

    try:
        wikipedia.set_lang(message.command[1])
        await message.reply_text("Lingua cambiata con successo!")
    except:
        await message.reply_text("Mi sa... che hai inserito una lingua che non esiste")


#!/usr/bin/env python3

import numpy as np
import time
import cv2
import matplotlib.pyplot as plt

def corr(F, I):
    """
    Input
        F: A (k, ell, c)-shaped ndarray containing the k x ell filter (with c channels).
        I: An (m, n, c)-shaped ndarray containing the m x n image (with c channels).

    Returns
        G: An (m, n)-shaped ndarray containing the correlation of the filter with the image.
    """
    ########## Code starts here ##########
    raise NotImplementedError("Implement me!")
    ########## Code ends here ##########

def norm_cross_corr(F, I):
    """
    Input
        F: A (k, ell, c)-shaped ndarray containing the k x ell filter (with c channels).
        I: An (m, n, c)-shaped ndarray containing the m x n image (with c channels).

    Returns
        G: An (m, n)-shaped ndarray containing the normalized cross-correlation of the filter with the image.
    """
    ########## Code starts here ##########
    raise NotImplementedError("Implement me!")
    ########## Code ends here ##########

def show_save_corr_img(filename, image, template):
    # Not super simple, because need to normalize image scale properly.
    fig, ax = plt.subplots()
    cropped_img = image[:-template.shape[0], :-template.shape[1]]
    im = ax.imshow(image, interpolation='none', vmin=cropped_img.min())
    fig.colorbar(im)
    fig.savefig(filename, bbox_inches='tight')
    plt.show()
    plt.close(fig)

def main():
    test_card = cv2.imread('test_card.png').astype(np.float32)

    filt1 = np.zeros((3, 3, 1))
    filt1[1, 1] = 1

    filt2 = np.zeros((3, 3, 1))
    filt2[1, -1] = 1

    filt3 = np.zeros((3, 3, 1))
    filt3[:, 0] = -1
    filt3[:, 2] = 1

    filt4 = (1./273.)*np.array([[1, 4, 7, 4, 1],
                              [4, 16, 26, 16, 4],
                              [7, 26, 41, 26, 7],
                              [4, 16, 26, 16, 4],
                              [1, 4, 7, 4, 1]])
    filt4 = np.expand_dims(filt4, -1)

    grayscale_filters = [filt1, filt2, filt3, filt4]

    color_filters = list()
    for filt in grayscale_filters:
        # Making color filters by replicating the existing
        # filter per color channel.
        color_filters.append(np.concatenate([filt, filt, filt], axis=-1))

    for idx, filt in enumerate(color_filters):
        start = time.time()
        corr_img = corr(filt, test_card)
        stop = time.time()
        print('Correlation function runtime:', stop - start, 's')
        show_save_corr_img("corr_img_filt%d.png" % idx, corr_img, filt)

if __name__ == "__main__":
    main()


from . import bp
from flask.views import MethodView
from app import db
from app.schemas import JobQueryArgsSchema, JobSchema
from app.models import Job

@bp.route('/jobs')
class Jobs(MethodView):
    @bp.arguments(JobQueryArgsSchema, location="query")
    @bp.response(JobSchema(many=True))
    @bp.paginate()
    def get(self, args, pagination_parameters):
        """List all jobs currently available"""
        data, total = Job.get(args, pagination_parameters.page,
                              pagination_parameters.page_size)
        pagination_parameters.item_count = total
        return data

    @bp.arguments(JobSchema)
    @bp.response(JobSchema)
    def post(self, job):
        """Post a new job with tags"""
        db.session.add(job)
        db.session.commit()
        return job


import config
import csv
import datetime
import main
import multiprocessing
import psycopg2
import time

sp = main.sp
cur = main.cur
lock = main.lock

def available_genres():
    try:
        print('Available genres for recommendation was requested!')
        results = sp.recommendation_genre_seeds()['genres']
        print('Available genres: ' + (', '.join(results)))
        return results

    except Exception as e:
        print('Available genres ERROR!')
        print(e)

def genre_song_label():
    try:
        row = []
        songs = get_songs_from_db()
        f = open('../datasets/labeled_songs.csv', 'w')

        with f:
            writer = csv.writer(f)
            for song in songs:
                artist_id = song[15]
                print('artist_id ' + artist_id)
                cur.execute("SELECT * FROM artists WHERE id=(%s);", (artist_id, ))
                artist = cur.fetchone()
                artist_genre = artist[2]

                if (artist_genre is None or artist_genre in ['', '[]', '{}']):
                    print('Artist genre is NULL!')
                    continue

                row = [artist_genre, str(song[0]), str(song[1]), str(song[2]), str(song[3]), str(song[4]),
                       str(song[5]), str(song[6]), str(song[7]), str(song[8]), str(song[9]), str(song[10]),
                       str(song[11]), str(song[12]), str(song[13]), str(song[14])]

                writer.writerow(row)

    except Exception as e:
        print('Song labeling ERROR!')
        print(e)

def get_artist(artist_id):
    try:
        print('Artist information for ' + artist_id + ' was requested!')
        artist_obj = sp.artist(artist_id)

        if (artist_obj == None):
            print('Artist information returned NULL!')
            return None

        print('Information for ' + artist_obj['name'] + ' was successfully fetched!')
        return artist_obj

    except Exception as e:
        print('Artist information ERROR!')
        print(e)

def get_album(album_id):
    try:
        print('Album information for ' + album_id + ' was requested!')
        album_obj = sp.album(album_id)

        if (album_obj == None):
            print('Album information returned NULL!')
            return None

        print('Information for ' + album_obj['name'] + ' was successfully fetched!')
        return album_obj

    except Exception as e:
        print('Artist information ERROR!')
        print(e)

def get_albums_by_artist(artist_id):
    try:
        print('Album information for ' + artist_id + ' was requested!')
        album_obj = sp.artist_albums(artist_id, album_type='album', limit=50)

        if (album_obj == None):
            print('Album information returned NULL!')
            return None

        for single_album in album_obj['items']:
            print('Information for ' + single_album['name'] + ' was successfully fetched!')

        return album_obj

    except Exception as e:
        print('Album information by artist ERROR!')
        print(e)

def get_songs_by_album(album_id):
    try:
        print('Song information for ' + album_id + ' was requested!')
        song_obj = sp.album_tracks(album_id, limit=50)

        if (song_obj == None):
            print('Song information returned NULL!')
            return None

        for single_song in song_obj['items']:
            print('Information for ' + single_song['name'] + ' was successfully fetched!')

        return song_obj

    except Exception as e:
        print('Song information ERROR!')
        print(e)

def get_song_features(song_id):
    try:
        audio_features = {}
        print('Song features for ' + song_id + ' was requested!')
        audio_obj = sp.audio_features(song_id)

        if (audio_obj == None):
            print('Song features returned NULL!')
            return None

        audio_features['acousticness'] = audio_obj[0]['acousticness']
        audio_features['instrumentalness'] = audio_obj[0]['instrumentalness']
        audio_features['valence'] = audio_obj[0]['valence']
        audio_features['loudness'] = audio_obj[0]['loudness']
        audio_features['energy'] = audio_obj[0]['energy']
        audio_features['liveness'] = audio_obj[0]['liveness']
        audio_features['danceability'] = audio_obj[0]['danceability']
        audio_features['tempo'] = audio_obj[0]['tempo']
        audio_features['speechiness'] = audio_obj[0]['speechiness']
        audio_features['mode'] = audio_obj[0]['mode']
        audio_features['duration_ms'] = audio_obj[0]['duration_ms']
        audio_features['time_signature'] = audio_obj[0]['time_signature']
        audio_features['key'] = audio_obj[0]['key']

        print('Song features for ' + song_id + ' was successfully fetched!')
        return audio_features

    except Exception as e:
        print('Song features ERROR!')
        print(e)

def get_artists_by_recommendation_genre(seeder):
    results = sp.recommendations(seed_genres=seeder, limit=50)
    results = results['tracks']
    results_pot = []

    for result in results:
        if (len(result['artists']) > 1):
            for single_artist in result['artists']:
                spotify_id = single_artist['id']
                lock.acquire()
                cur.execute("SELECT COUNT(*) FROM artists WHERE id=(%s);", (spotify_id, ))

                if (cur.fetchone()[0] == 1):
                    print(spotify_id + ' artist is already in DB.')
                    lock.release()
                    continue
                else:
                    lock.release()

                artist_obj = get_artist(spotify_id)
                if (not artist_obj):
                    print('Artist object is NULL!')
                    continue

                popularity = artist_obj['popularity']
                genres = artist_obj['genres']

                if (popularity < config.ENV['MIN_ARTIST_POP'] or (len(genres) == 0) or (genres is None)):
                    print('Not enough popularity or genre not found!')
                    continue

                name = single_artist['name']
                results_pot.append((spotify_id, name, genres, popularity))

        else:
            single_artist = result['artists'][0]
            spotify_id = single_artist['id']
            lock.acquire()
            cur.execute("SELECT COUNT(*) FROM artists WHERE id=(%s);", (spotify_id, ))

            if (cur.fetchone()[0] == 1):
                print(spotify_id + ' artist is already in DB.')
                lock.release()
                continue
            else:
                lock.release()

            artist_obj = get_artist(spotify_id)
            popularity = artist_obj['popularity']
            if (not artist_obj):
                print('Artist object is NULL!')
                continue
            if (popularity < config.ENV['MIN_ARTIST_POP']):
                continue

            name = single_artist['name']
            genres = artist_obj['genres']
            results_pot.append((spotify_id, name, genres, popularity))

    return results_pot

def insert_artists_by_genres(combinated_genre):
    try:
        seeder = combinated_genre
        artists = get_artists_by_recommendation_genre(seeder)

        for artist in artists:
            try:
                lock.acquire()
                cur.execute("SELECT COUNT(*) FROM artists WHERE id=(%s);", (artist[0], ))

                if ((len(artist[2]) == 0) or (artist[2] is None)):
                    print(artist[0] + ' artist genre is NOT FOUND!')
                    lock.release()
                elif (cur.fetchone()[0] == 1):
                    print(artist[0] + ' artist is already in DB.')
                    lock.release()
                    continue
                else:
                    lock.release()

                lock.acquire()
                cur.execute("""INSERT INTO artists (
                    id, name, genres, popularity, created_at)
                    VALUES (%s, %s, %s, %s, %s)""",
                    (artist[0], artist[1], artist[2], artist[3], datetime.datetime.utcnow()))
                lock.release()

            except psycopg2.IntegrityError as e:
                print('Exception was captured but program is still in progress.')  # Mostly to prevent DB from duplicated PKs.
                print(e)
                lock.release()
                continue

            except Exception as e:
                print(e)
                continue

    except KeyboardInterrupt:
        print('Process was interrupted!')
        return

def get_and_insert_artists(combinated_genres):
    try:
        num_active_processes = 0
        for single_combinated_genre in combinated_genres:
            concatenated_genre = ("-".join(single_combinated_genre))
            lock.acquire()
            cur.execute("SELECT COUNT(*) FROM combinated_genres WHERE name=(%s);", (concatenated_genre, ))

            if (cur.fetchone()[0] == 1):
                print("-".join(single_combinated_genre) + ' combinated genre was already fetched!')
                lock.release()
                continue
            else:
                lock.release()

            if (num_active_processes < config.ENV['NUM_PROCESSES']):
                p = multiprocessing.Process(target=insert_artists_by_genres, args=(single_combinated_genre,))
                p.daemon = True
                p.start()
                num_active_processes += 1
            else:
                insert_artists_by_genres(single_combinated_genre)
                time.sleep(config.ENV['MULTIPROCESS_SLEEP'])
                num_active_processes = 0

            lock.acquire()
            cur.execute("""INSERT INTO combinated_genres (
                    name, created_at)
                    VALUES (%s, %s)""",
                    (concatenated_genre, datetime.datetime.utcnow()))
            print("-".join(single_combinated_genre) + ' combinated genre was written in DB!')
            lock.release()

    except KeyboardInterrupt:
        print('User interrupted the process!')

    except Exception as e:
        print(e)

def insert_albums(artist):
    try:
        for album in get_albums_by_artist(artist[0])['items']:
            if (album == None):
                continue

            album_obj = get_album(album['id'])

            if (album_obj == None):
                continue

            album_id = album_obj['id']
            lock.acquire()
            cur.execute("SELECT COUNT(*) FROM albums WHERE id=(%s);", (album_id, ))

            if (cur.fetchone()[0] == 1):
                print(album_id + ' album is already in DB.')
                lock.release()
                continue
            else:
                lock.release()

            album_name = album_obj['name']
            # album_genres = album_obj['genres']
            total_tracks = album_obj['total_tracks']
            artist_id = artist[0]
            artist_name = artist[1]

            lock.acquire()
            cur.execute("""INSERT INTO albums (
                    id, name, total_tracks, artist_id, artist_name, created_at)
                    VALUES (%s, %s, %s, %s, %s, %s)""",
                    (album_id, album_name, total_tracks, artist_id, artist_name, datetime.datetime.utcnow()))
            print(album_id + ' album was written in DB.')
            lock.release()

    except KeyboardInterrupt:
        print('User interrupted the process!')

    except Exception as e:
        print(e)

def insert_songs(album):
    try:
        for song in get_songs_by_album(album[0])['items']:

            if (song == None):
                continue

            song_id = song['id']
            lock.acquire()
            cur.execute("SELECT COUNT(*) FROM songs WHERE id=(%s);", (song_id, ))

            if (cur.fetchone()[0] == 1):
                print(song_id + ' song is already in DB.')
                lock.release()
                continue
            else:
                lock.release()

            song_name = song['name']
            song_feature_obj = get_song_features(song_id)

            if (song_feature_obj == None):
                continue

            acousticness = song_feature_obj['acousticness']
            instrumentalness = song_feature_obj['instrumentalness']
            valence = song_feature_obj['valence']
            loudness = song_feature_obj['loudness']
            energy = song_feature_obj['energy']
            liveness = song_feature_obj['liveness']
            danceability = song_feature_obj['danceability']
            tempo = song_feature_obj['tempo']
            speechiness = song_feature_obj['speechiness']
            mode = song_feature_obj['mode']
            duration_ms = song_feature_obj['duration_ms']
            time_signature = song_feature_obj['time_signature']
            key = song_feature_obj['key']

            artist_id = album[3]
            artist_name = album[4]
            album_id = album[0]
            album_name = album[1]

            lock.acquire()
            cur.execute("""INSERT INTO songs (
                    id, name,
                    acousticness, instrumentalness, valence, loudness, energy, liveness, danceability, tempo, speechiness,
                    mode, duration_ms, time_signature, key,
                    artist_id, artist_name, album_id, album_name,
                    created_at)
                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
                    (song_id, song_name,
                    acousticness, instrumentalness, valence, loudness, energy, liveness, danceability, tempo, speechiness,
                    mode, duration_ms, time_signature, key,
                    artist_id, artist_name, album_id, album_name,
                    datetime.datetime.utcnow()))
            print(song_id + ' song was written in DB.')
            lock.release()

    except KeyboardInterrupt:
        print('User interrupted the process!')

    except Exception as e:
        print(e)

def insert_predictions(prediction):
    try:
        if (prediction == None):
            print('Prediction is NONE!')
            return

        song_id = prediction['id']
        song_name = prediction['name']
        r_b = float(prediction['probs'][0][0])
        rap = float(prediction['probs'][0][1])
        electronic = float(prediction['probs'][0][2])
        rock = float(prediction['probs'][0][3])
        new_age = float(prediction['probs'][0][4])
        classical = float(prediction['probs'][0][5])
        reggae = float(prediction['probs'][0][6])
        blues = float(prediction['probs'][0][7])
        country = float(prediction['probs'][0][8])
        world = float(prediction['probs'][0][9])
        folk = float(prediction['probs'][0][10])
        easy_listening = float(prediction['probs'][0][11])
        jazz = float(prediction['probs'][0][12])
        vocal = float(prediction['probs'][0][13])
        punk = float(prediction['probs'][0][14])
        alternative = float(prediction['probs'][0][15])
        pop = float(prediction['probs'][0][16])
        heavy_metal = float(prediction['probs'][0][17])

        lock.acquire()
        cur.execute("""INSERT INTO song_genre_predictions (
                    id, name, r_b, rap, electronic, rock, new_age, classical, reggae, blues, country, world, folk, easy_listening, jazz, vocal, punk, alternative, pop, heavy_metal, created_at)
                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
                    (song_id, song_name, r_b, rap, electronic, rock, new_age, classical, reggae, blues, country, world, folk, easy_listening, jazz, vocal, punk, alternative, pop, heavy_metal, datetime.datetime.utcnow()))
        print(song_id + ' song was written in DB.')
        lock.release()

    except KeyboardInterrupt:
        print('User interrupted the process!')

    except Exception as e:
        print(e)

def plt_predictions():
    # Sample prediction bar charts for genre classifier model results.
    # Save results as 'png' in 'figures' folder.
    # Radiohead - Alternative/Rock
    # Justin Bieber - Pop
    # Nirvana - Rock
    # 6ix9ine - Rap

    genres = ['r_b', 'rap', 'electronic', 'rock', 'new_age', 'classical', 'reggae', 'blues', 'country',
    'world', 'folk', 'easy_listening', 'jazz', 'vocal', 'punk', 'alternative', 'pop', 'heavy_metal']
    artist_names = ['radiohead', 'justin_bieber', 'nirvana', '6ix9ine']
    artist_id = {
        'radiohead': '4Z8W4fKeB5YxbusRsdQVPb',
        'justin_bieber': '1uNFoZAHBGtllmzznpCI3s',
        'nirvana': '6olE6TJLqED3rqDCT0FyPh',
        '6ix9ine': '7gZfnEnfiaHzxARJ2LeXrf'
    }

    for artist_name in artist_names:
        songs = get_songs_of_artist_from_db(artist_id[artist_name])
        song_ids = [x[0] for x in songs]
        for song_id in song_ids:
            prediction = get_genre_predictions_from_db(song_id)
            song_name = prediction[0][1].replace('/', '_').replace('.', '_')
            prediction = [[x[2:20] for x in prediction]]
            prediction = prediction[0][0]
            plt.subplots(num=None, figsize=(18, 10), dpi=60, facecolor='w', edgecolor='k')
            plt.xlabel('Genre Correlation Value')
            plt.ylabel('Genres')
            plt.barh(genres, prediction)
            plt.title(song_name)
            plt.savefig('figures/' + artist_name + '/' + song_name + '-' + song_id)
            # plt.show()
            plt.clf()
            plt.close()

def get_artists_from_db():
    cur.execute("SELECT * FROM artists ORDER BY RANDOM()")
    return cur.fetchall()

def get_albums_from_db():
    cur.execute("SELECT * FROM albums ORDER BY RANDOM()")
    return cur.fetchall()

def get_songs_from_db():
    cur.execute("SELECT * FROM songs ORDER BY RANDOM()")
    return cur.fetchall()

def get_count_songs_from_db():
    cur.execute("SELECT COUNT(*) FROM songs")
    return cur.fetchall()[0][0]

def artists_genre_occurrences():
    cur.execute("SELECT genres, COUNT(genres) FROM artists GROUP BY genres ORDER BY COUNT(genres) DESC")
    return cur.fetchall()

def get_genre_predictions_from_db(song_id):
    cur.execute("SELECT * FROM song_genre_predictions WHERE id=%s", (song_id,))
    return cur.fetchall()

def get_songs_of_artist_from_db(artist_id):
    cur.execute("SELECT * FROM songs WHERE artist_id=%s", (artist_id,))
    return cur.fetchall()


import json
import time
import os
import datetime

from scraper.halooglasi import get_latest_with_retry
from scraper.db import check_if_exists, insert_property
from scraper.discord import send_to_discord
from scraper.logger import logger

def get_local_time():
    """
    convert current time to CEST
    """
    cest_time = datetime.datetime.now() + datetime.timedelta(hours=2)
    return cest_time.strftime("%D %H:%M:%S")

def main():
    CFG_FILE = os.environ.get("CFG_FILE", "/app/config.json")
    # in seconds
    SLEEP_TIME = float(os.environ.get("SLEEP_TIME", 30))

    config = []
    while True:
        with open(CFG_FILE, "r") as f:
            _config = json.load(f)
            if _config != config:
                logger.info(f"New config: {_config}")
                config = _config

        for url, hook_url in config.items():
            logger.info(f"Scraping... {get_local_time()}")
            results = get_latest_with_retry(url, max_retries=5, sleep_time=SLEEP_TIME)
            for link, price, name, location in results[::-1]:
                if not check_if_exists(link):
                    insert_property(link, price, name, location)
                    logger.info(f"Inserted {name} with price {price}")
                    send_to_discord(link, price, name, location, hook_url=hook_url)
            time.sleep(SLEEP_TIME)

if __name__ == "__main__":
    main()


from flask_app.config.mysqlconnection import connectToMySQL
from flask import flash
from flask_app.controllers.users import User

class Sighting:
    def __init__(self, data):
        self.id = data['id']
        self.location = data['location']
        self.scenario = data['scenario']
        self.date_of_sighting = data['date_of_sighting']
        self.qty_of_sq = data['qty_of_sq']
        self.created_at = data['created_at']
        self.updated_at = data['updated_at']
        self.user = None
        self.skeptic_user = []

    @classmethod
    def all_sightings(cls):
        query = """
                SELECT * FROM sightings
                JOIN users on sightings.user_id = users.id;
                """
        results = connectToMySQL('sasquatch2').query_db(query)
        sightings = []
        if not results:
            return sightings
        for info in results:
            sighting = cls(results[0])
            row = {
                'id': info['users.id'],
                'first_name': info['first_name'],
                'last_name': info['last_name'],
                'email': info['email'],
                'password': "",
                'created_at': info['users.created_at'],
                'updated_at': info['users.updated_at']

            }
            sighting.user = User(row)
            sightings.append(sighting)
        return sightings

    @classmethod
    def add_sighting(cls,data):
        query = """
                INSERT INTO sightings(location, scenario, date_of_sighting, qty_of_sq, created_at, updated_at, user_id)
                VALUES(%(location)s, %(scenario)s, %(date_of_sighting)s, %(qty_of_sq)s,NOW(), NOW(), %(user_id)s);
                """
        results = connectToMySQL('sasquatch2').query_db(query,data)
        return results

    @classmethod
    def getidby(cls,data):
        query = """
                SELECT * FROM sightings
                WHERE id = %(id)s;
                """
        results = connectToMySQL('sasquatch2').query_db(query,data)
        return cls(results[0])

    @classmethod
    def update_sightings(cls,data):
        query = """
                UPDATE sightings
                SET location = %(location)s, scenario = %(scenario)s, date_of_sighting = %(date_of_sighting)s, qty_of_sq = %(qty_of_sq)s, updated_at = NOW()
                WHERE id = %(id)s;
                """
        results = connectToMySQL('sasquatch2').query_db(query,data)
        return results

    @staticmethod
    def validate_sighting(sighting_info):
        is_valid = True

        if len(sighting_info['location']) < 2:
            flash("Location is needed/Not valid location")
            is_valid = False
        if len(sighting_info['scenario']) < 2:
            flash("Tell us what happened/Not a valid entry")
            is_valid = False
        if sighting_info['qty_of_sq'] == ''  :
            flash("Please tell us how many you saw")
            is_valid = False
        if sighting_info['date_of_sighting'] == ''  :
            flash("Please input a date")
            is_valid = False

        return is_valid

    @classmethod
    def remove_sighting(cls, data):
        query = """
                DELETE FROM sightings
                WHERE id = %(id)s
                """
        results = connectToMySQL('sasquatch2').query_db(query,data)
        return results

    @classmethod
    def skeptic_sight(cls):
        query = """
                SELECT * FROM sightings
                LEFT JOIN skeptics ON sightings.id = skeptics.sighting_id
                LEFT JOIN users ON users.id = skeptics.user_id
                WHERE sightings.id = %(id)s;
                """
        results = connectToMySQL('sasquatch2').query_db(query)
        sightings = []
        if not results:
            return sightings
        for sight in results:
            if sight['users.id'] == None:
                break
            sighting = cls(results[0])
            data ={
                'id': sight['users.id'],
                'first_name': sight['first_name'],
                'last_name': sight['last_name'],
                'email': sight['email'],
                'password': "",
                'created_at': sight['users.created_at'],
                'updated_at': sight['users.updated_at']
            }
            sighting.user= User(data)
            sightings.append(sighting)
        return sightings


from rest_framework import generics
from rest_framework.response import Response
from .models import Ganado, Categoria, Raza, LitrosDeLeche
from .serializers import GanadoSerializer,CategoriaSerializer, RazaSerializer, LitrosDeLecheSerializer
from django.views.generic import View

class GanadoListCreateAPIView(generics.ListCreateAPIView):
    queryset = Ganado.objects.all()
    serializer_class = GanadoSerializer

class GanadoRetrieveUpdateDestroyView(generics.RetrieveUpdateDestroyAPIView):
    queryset = Ganado.objects.all()
    serializer_class = GanadoSerializer

class GanadoDestroyAPIView(generics.DestroyAPIView):
    queryset = Ganado.objects.all()
    serializer_class = GanadoSerializer

class GanadoSearchView(View):
    def get(self, request):
        search_query = request.GET.get('search')
        if search_query:
            ganados = Ganado.objects.filter(nombre__icontains=search_query)
        else:
            ganados = Ganado.objects.all()
        return render(request, 'ganado_search.html', {'ganados': ganados, 'search_query': search_query})

    #categoria

class CategoriaListCreateAPIView(generics.ListCreateAPIView):
    queryset = Categoria.objects.all()
    serializer_class = CategoriaSerializer

class CategoriaRetrieveUpdateDestroyView(generics.RetrieveUpdateDestroyAPIView):
    queryset = Categoria.objects.all()
    serializer_class = CategoriaSerializer

class CategoriaDestroyAPIView(generics.DestroyAPIView):
    queryset = Categoria.objects.all()
    serializer_class = CategoriaSerializer

class CategoriaSearchView(View):
    def get(self, request):
        search_query = request.GET.get('search')
        if search_query:
            categorias = Categoria.objects.filter(nombre__icontains=search_query)
        else:
            categorias = Categoria.objects.all()
        return render(request, 'ganado_search.html', {'ganados': categorias, 'search_query': search_query})

    #Raza

class RazaListCreateAPIView(generics.ListCreateAPIView):
    queryset = Raza.objects.all()
    serializer_class = RazaSerializer

class RazaRetrieveUpdateDestroyView(generics.RetrieveUpdateDestroyAPIView):
    queryset = Raza.objects.all()
    serializer_class = RazaSerializer

class RazaDestroyAPIView(generics.DestroyAPIView):
    queryset = Raza.objects.all()
    serializer_class = RazaSerializer

#litros de leche
class LitrosDeLecheListCreateAPIView(generics.ListCreateAPIView):
    queryset = LitrosDeLeche.objects.all()
    serializer_class = LitrosDeLecheSerializer

class LitrosDeLecheRetrieveUpdateDestroyView(generics.RetrieveUpdateDestroyAPIView):
    queryset = LitrosDeLeche.objects.all()
    serializer_class = LitrosDeLecheSerializer

class LitrosDeLecheDestroyAPIView(generics.DestroyAPIView):
    queryset = LitrosDeLeche.objects.all()
    serializer_class = LitrosDeLecheSerializer


import util
from command import process_command
import session as discord_session
import filters
from discord import Embed
from logger import log

# for the memes
from random import choice

@util.client.event
async def on_message(message):
    if message.author.bot:
        # check to make sure it didn't send message
        if message.author == util.client.user:
            return

        # brandon clauses
        if message.content.startswith('What do you call a'):
            await insult(message)
        elif message.content.startswith('How do you'):
            await insult(message)
        return

    try:
        await handle_message(message)
    except Exception as e:
        log(message.server.id, str(e))
        await util.client.send_message(message.channel, embed=Embed(color=util.error_color, description='An unknown error occurred.'))

async def insult(message):
    messages = [
        'That joke wasn\'t funny and it never will be.',
        'You are not funny.',
        'Please be quiet for all of our sakes.',
        'You make me want to die.',
        'You don\'t need to tell jokes; you are one.',
        'Inferior...',
        ':joy: :gun:',
        '```Roses are red,\nViolets are blue,\nThat bot is garbage,\nand Brandon is too.```'
    ]
    await util.client.send_message(message.channel, choice(messages))

@util.client.event
async def on_server_join(server):
    embed = Embed(color=util.theme_color, title='Null Bot')
    embed.description = 'Null is a fully featured, powerful, multipurpose Discord bot from easy use on any server.'
    embed.add_field(name='Get Started', value='Use `!help` to get list of command and how to use them.', inline=False)
    embed.add_field(name='Support', value='**Donate** donate_url\n**Source** github_url\n**Upvote** upvote_url', inline=False)
    embed.add_field(name='Join our Discord', value='discord_url', inline=False)
    icon_url = 'https://cdn.discordapp.com/avatars/226732838181928970/19562db0c14f445ac5a0bf8f605989c1.png?size=128'
    embed.set_footer(text='Developed by ComedicChimera#3451', icon_url=icon_url)
    await util.client.send_message(server.default_channel, embed=embed)

@util.client.event
async def on_member_join(member):
    if member.id in util.servers[member.server].hack_bans:
        await util.client.kick(member)
    else:
        await util.client.send_message(member.server.default_channel, 'Welcome `%s`!' % member.name)

async def handle_message(message):
    prefix = util.get_server_prefix(message.server)
    # use custom input handler if specified
    if discord_session.has_session(message.server, message.channel, message.author):
        await util.client.send_typing(message.channel)
        s = discord_session.get_session(message.server, message.channel, message.author)
        await s.handler(message, s.session_id)
    # else pass to command handler
    elif message.content.startswith(prefix):
        await process_command(message, prefix)
    # apply filters
    elif filters.match_filters(message):
        await util.client.delete_message(message)

if __name__ == '__main__':
    # imported command sets
    import modules.general.commands
    import modules.music.commands
    import modules.math.commands
    import modules.internet.commands
    import modules.money.commands
    import modules.games.commands
    import modules.admin.commands

    # start the bot
    util.client.run(util.token)


from mrjob.job import MRJob
from mrjob.protocol import JSONProtocol, RawValueProtocol, JSONValueProtocol
from mrjob.step import MRStep
import json
import numpy as np

def multivar_gauss_pdf(x, mu, cov):
    '''
    Caculates the multivariate normal density (pdf)

    Parameters:
    -----------

    x    - numpy array of a "d x 1" sample vector
    mu   - numpy array of a "d x 1" mean vector
    cov  - numpy array of a d x d" covariance matrix

    (where d - dimensionality of data)

    Output:
    -------
            - (float) probability of x given parameters of
                     Gaussian Distribution
    '''
    part1 = 1 / ( ((2* np.pi)**(len(mu)/2)) * (np.linalg.det(cov)**(1/2)) )
    part2 = (-1/2) * np.dot(np.dot((x-mu).T,(np.linalg.inv(cov))),(x-mu))
    return float(part1 * np.exp(part2))

def responsibility(x,mu,cov,p,K):
    '''
    Calculates conditional probability of latent variable given
    observed data and parameters

    Parameters:
    -----------

    x     - numpy array of a "d x 1" sample vector
    mu    - list of length "K" of lists "d x 1" mean vector
    cov   - list of length "K" numpy arrays each "d x d" covariance matrix
    p     - list of floats, each float prior probability of cluster
    K     - number of clusters (values of latent variables)

    (where d - dimensionality of data)

    Output:
          - list of floats, each element of list is responsibility corresponding
            to x and relevant latent variable valiue
    '''
    resps = [p[k]*multivar_gauss_pdf(x,np.array(mu[k]),np.array(cov[k])) for k in range(K)]
    p_x = sum(resps)
    return [float(r_k)/p_x for r_k in resps]

def extract_features(line):
    ''' extracts features from line of input'''
    data = line.strip().split(",")
    return [ float(e) for e in data[1:] ]

def make_json_encodable(mixing, means, covar):
    '''
    Transforms

    Parameters:
    -----------

    mixing   - list of size k
    means    - list of size k of numpy arrays (each numpy array has size d)
    covar    - list of size k of two dimensional numpy array (matrix of size dxd)

    (where d is dimensionality and k is number of clusters)

    Output:
    --------
             - dictionary with parameter names as keys
             {"mu": list of mean vectors, "mixing": list of mixing coefficients,
              "covariance": list of covariance matrices}

    '''
    matrix_to_list = lambda x: [list(e) for e in x]
    mixing = mixing
    means = matrix_to_list(means)
    covariance = [matrix_to_list(e) for e in covar]
    return {"mixing":mixing,"mu":means,"covariance":covariance}

class IterationGaussianMixtureMR(MRJob):
    '''
    Runs single iteration of Expectation Maximization Algorithm for Gaussian
    Mixture Model.

    Mappers use parameters from previous iteration to calculate responsibilities
    and intermediate values that are then used by single reducer to calculate
    new parameters.

    Command Line Options:
    ---------------------

    --clusters             - number of clusters
    --dimensions           - dimensionality of data
    --parameters           - (str)json encoded dictionary of parameters

    '''
    INPUT_PROTOCOL = RawValueProtocol

    INTERNAL_PROTOCOL = JSONProtocol

    OUTPUT_PROTOCOL = JSONValueProtocol

    def __init__(self,*args,**kwargs):
        super(IterationGaussianMixtureMR,self).__init__(*args,**kwargs)
        # sum of responsibilities for each cluster & number of observations
        self.resp_sum = [0]*self.clusters
        self.N = 0
        # sum of observations weighted by reponsibility
        self.resp_w_sum = [np.zeros(self.dim, dtype = np.float64) for i in range(self.clusters)]
        # sum of x_n*x_n_t (outer products) weighted by reponsibility
        self.resp_w_cov = [np.zeros([self.dim,self.dim], dtype = np.float64) for i in range(self.clusters)]

    def configure_options(self):
        super(IterationGaussianMixtureMR,self).configure_options()
        self.add_passthrough_option("--dimensions",
                                    type = int,
                                    help = "dimensionality of input data")
        self.add_passthrough_option("--clusters",
                                    type = int,
                                    help = "number of clusters")
        self.add_passthrough_option("--parameters",
                             type = str,
                             help = "file with parameters from previous iteration")

    def load_options(self,args):
        super(IterationGaussianMixtureMR,self).load_options(args)
        # number of clusters
        if self.options.clusters is None:
            self.option_parser.error("You need to specify number of clusters")
        else:
            self.clusters = self.options.clusters
        # data dimensionality
        if self.options.dimensions is None:
            self.option_parser.error("You need to specify dimensionality of data")
        else:
            self.dim = self.options.dimensions
        # filename where parameters from previous iteration are saved
        if self.options.parameters is None:
            self.option_parser.error("You need to load file with distribution parameters")

    def mapper_gmm_init(self):
        params = json.loads(self.options.parameters)
        self.mu = params["mu"]
        self.covar = params["covariance"]
        self.mixing = params["mixing"]

    def mapper_gmm(self,_,line):
        features = extract_features(line)
        assert(len(features)==self.dim), "dimension mismatch"
        x = np.array(features)
        r_n = responsibility(x,self.mu,self.covar,self.mixing,self.clusters) # responsibilities
        self.resp_sum = [self.resp_sum[i]+r_n_k for i,r_n_k in enumerate(r_n)]
        self.resp_w_sum = [w_sum + r_n[i]*x for i,w_sum in enumerate(self.resp_w_sum)]
        self.resp_w_cov = [w_covar+r_n[i]*np.outer(x,x) for i,w_covar in enumerate(self.resp_w_cov)]
        self.N+=1

    def mapper_final_gmm(self):
        matrix_to_list = lambda x: [list(e) for e in x]
        # sum of responsibilities
        yield 1,("r_sum", self.resp_sum)
        # sum of observations weighted by responsibility
        yield 1,("r_w_sum", [list(e) for e in self.resp_w_sum])
        # covariates weighted by responsibility
        yield 1,("r_w_cov", [ matrix_to_list(cov) for cov in self.resp_w_cov])
        # number of observations
        yield 1,("total", self.N)

    def reducer_gmm(self,key, values):
        N = 0;
        r_sum = [0]*self.clusters
        r_w_sum = [np.zeros(self.dim, dtype = np.float64) for i in range(self.clusters)]
        r_w_cov = [np.zeros([self.dim,self.dim], dtype = np.float64) for i in range(self.clusters)]
        for value in values:
            if value[0]=="r_sum":
                r_sum = [r_sum[i]+gamma for i,gamma in enumerate(value[1])]
            elif value[0]=="r_w_sum":
                r_w_sum = [r_w_sum[i]+np.array(r_w_new, dtype = np.float64) for i,r_w_new in enumerate(value[1])]
            elif value[0]=="r_w_cov":
                r_w_cov = [ r_w_cov[i] + np.array(cov) for i,cov in enumerate(value[1])]
            elif value[0]=="total":
                N+=value[1]
        mixing = [float(gamma)/N for gamma in r_sum]
        means =  [1.0/r_sum[i]*r_w_sum[i] for i, gamma in enumerate(mixing)]
        covar =  [ 1.0/r_sum[k]*r_w_cov_k - np.outer(means[k],means[k]) for k,r_w_cov_k in enumerate(r_w_cov)]
        yield None, make_json_encodable(mixing,means,covar)

    def steps(self):
        return [MRStep(mapper_init = self.mapper_gmm_init,
                       mapper = self.mapper_gmm,
                       mapper_final = self.mapper_final_gmm,
                       reducer = self.reducer_gmm)]

if __name__=="__main__":
    IterationGaussianMixtureMR.run()

Gaussian Mixture Model MapReduce/IterationGaussianMixtureMR.py


import torch
import torch.nn as nn
import torch.nn.functional as F

from pycls.models.resnet_style.shake_shake_function import get_alpha_beta, shake_function

import pycls.utils.logging as lu

logger = lu.get_logger(__name__)

def initialize_weights(module):
    if isinstance(module, nn.Conv2d):
        nn.init.kaiming_normal_(module.weight.data, mode='fan_out')
    elif isinstance(module, nn.BatchNorm2d):
        module.weight.data.fill_(1)
        module.bias.data.zero_()
    elif isinstance(module, nn.Linear):
        module.bias.data.zero_()

class ResidualPath(nn.Module):
    def __init__(self, in_channels, out_channels, stride):
        super(ResidualPath, self).__init__()

        self.conv1 = nn.Conv2d(
            in_channels,
            out_channels,
            kernel_size=3,
            stride=stride,
            padding=1,
            bias=False,
        )
        self.bn1 = nn.BatchNorm2d(out_channels)
        self.conv2 = nn.Conv2d(
            out_channels,
            out_channels,
            kernel_size=3,
            stride=1,
            padding=1,
            bias=False)
        self.bn2 = nn.BatchNorm2d(out_channels)

    def forward(self, x):
        x = F.relu(x, inplace=False)
        x = F.relu(self.bn1(self.conv1(x)), inplace=False)
        x = self.bn2(self.conv2(x))
        return x

class DownsamplingShortcut(nn.Module):
    def __init__(self, in_channels):
        super(DownsamplingShortcut, self).__init__()
        self.conv1 = nn.Conv2d(
            in_channels,
            in_channels,
            kernel_size=1,
            stride=1,
            padding=0,
            bias=False)
        self.conv2 = nn.Conv2d(
            in_channels,
            in_channels,
            kernel_size=1,
            stride=1,
            padding=0,
            bias=False)
        self.bn = nn.BatchNorm2d(in_channels * 2)

    def forward(self, x):
        x = F.relu(x, inplace=False)
        y1 = F.avg_pool2d(x, kernel_size=1, stride=2, padding=0)
        y1 = self.conv1(y1)

        y2 = F.pad(x[:, :, 1:, 1:], (0, 1, 0, 1))
        y2 = F.avg_pool2d(y2, kernel_size=1, stride=2, padding=0)
        y2 = self.conv2(y2)

        z = torch.cat([y1, y2], dim=1)
        z = self.bn(z)

        return z

class BasicBlock(nn.Module):
    def __init__(self, in_channels, out_channels, stride, shake_config):
        super(BasicBlock, self).__init__()

        self.shake_config = shake_config

        self.residual_path1 = ResidualPath(in_channels, out_channels, stride)
        self.residual_path2 = ResidualPath(in_channels, out_channels, stride)

        self.shortcut = nn.Sequential()
        if in_channels != out_channels:
            self.shortcut.add_module('downsample',
                                     DownsamplingShortcut(in_channels))

    def forward(self, x):
        x1 = self.residual_path1(x)
        x2 = self.residual_path2(x)

        if self.training:
            shake_config = self.shake_config
        else:
            shake_config = (False, False, False)

        alpha, beta = get_alpha_beta(x.size(0), shake_config, x.device)
        y = shake_function(x1, x2, alpha, beta)

        return self.shortcut(x) + y

class Network(nn.Module):
    def __init__(self, cfg):
        super(Network, self).__init__()

        print("Building Network of resnet Shake-Shake")

        self.cfg = cfg

        input_shape = (1, self.cfg.TRAIN.IM_CHANNELS, self.cfg.TRAIN.IM_SIZE, \
             self.cfg.TRAIN.IM_SIZE)

        print(f"Input_shape: {input_shape}")
        # input_shape = (1,3,224,224)

        n_classes = self.cfg.MODEL.NUM_CLASSES

        base_channels = self.cfg.SHAKE_SHAKE.BASE_CHANNELS
        #config['base_channels']
        depth = self.cfg.SHAKE_SHAKE.DEPTH
        #config['depth']
        self.shake_config = (self.cfg.SHAKE_SHAKE.FORWARD, self.cfg.SHAKE_SHAKE.BACKWARD, self.cfg.SHAKE_SHAKE.IMAGE)
        #(config['shake_forward'], config['shake_backward'],config['shake_image'])

        block = BasicBlock

        n_blocks_per_stage = (depth - 2) // 6
        assert n_blocks_per_stage * 6 + 2 == depth, f"Condition (n_blocks_per_stage * 6 + 2) == model_depth fails as model_depth: {model_depth} \
            and n_blocks_per_stage*6 + 2 = {n_blocks_per_stage*6 + 2}"

        n_channels = [base_channels, base_channels * 2, base_channels * 4]

        self.conv = nn.Conv2d(
            input_shape[1],
            n_channels[0],
            kernel_size=3,
            stride=1,
            padding=1,
            bias=False)

        self.bn = nn.BatchNorm2d(base_channels)

        self.stage1 = self._make_stage(
            n_channels[0], n_channels[0], n_blocks_per_stage, block, stride=1)
        self.stage2 = self._make_stage(
            n_channels[0], n_channels[1], n_blocks_per_stage, block, stride=2)
        self.stage3 = self._make_stage(
            n_channels[1], n_channels[2], n_blocks_per_stage, block, stride=2)

        # # compute conv feature size

        # with torch.no_grad():
        #     self.feature_size = self._forward_conv(
        #         torch.zeros(1,3,224,224)).view(-1).shape[0]

        self.feature_size = 128

        # if self.cfg.TRAIN.DATASET == "IMAGENET":
        #     self.feature_size = 128
        # else:
        #     self.feature_size = 128
        #     #raise NotImplementedError

        self.fc = nn.Linear(self.feature_size, n_classes)

        ###
        self.penultimate_active = False
        #Describe model with source code link
        self.description = "Open source implementation of Resnet shake-shake adapted from hysts/pytorch_shake_shake/ repository"

        self.source_link = "https://github.com/hysts/pytorch_shake_shake/blob/master/shake_shake.py"

        self.model_depth = cfg.MODEL.TRANSFER_MODEL_DEPTH if cfg.TRAIN.TRANSFER_EXP else cfg.MODEL.DEPTH

        logger.info('Constructing: Shake Shake ResNet with depth:{}'.format(self.model_depth))

        # initialize weights
        self.apply(initialize_weights)

    def _make_stage(self, in_channels, out_channels, n_blocks, block, stride):
        stage = nn.Sequential()
        for index in range(n_blocks):
            block_name = 'block{}'.format(index + 1)
            if index == 0:
                stage.add_module(
                    block_name,
                    block(
                        in_channels,
                        out_channels,
                        stride=stride,
                        shake_config=self.shake_config))
            else:
                stage.add_module(
                    block_name,
                    block(
                        out_channels,
                        out_channels,
                        stride=1,
                        shake_config=self.shake_config))
        return stage

    def _forward_conv(self, x):
        x = F.relu(self.bn(self.conv(x)), inplace=True)
        x = self.stage1(x)
        x = self.stage2(x)
        x = self.stage3(x)
        x = F.adaptive_avg_pool2d(x, output_size=1)
        return x

    def forward(self, x):
        x = self._forward_conv(x)
        z = x.view(x.size(0), -1)
        # print(f"~~z.shape[1]: {z.shape[1]}")
        x = self.fc(z)
        if self.penultimate_active:
            return z,x
        return x


#tire_volume.py
from datetime import datetime
import math
print('This program calculte the tire volume')
print()
form_1 = 0
volume_1 = None

w=float(input('Please Enter the width of the tire in mm: '))
a =float(input('Please Enter the aspect ratio of the tire: '))
d =float(input('Enter the diameter of the wheel in inches : '))
w1 =int(w)
a1 =int(a)
d1 =int(d)

volume_1 = math.pi*(w**2)*a*(w*a+2540*d)/10000000000

print('the volume is: {:,.2f}'.format(volume_1))
current_date_and_time = datetime.now()
print(f"{current_date_and_time:%Y-%m-%d}")
with open("tire_volume.txt",'a')as volume_file:
    print(file = volume_file)
    print(f"{current_date_and_time:%Y-%m-%d}, {w1}, {a1}, {d1}, {volume_1:,.2f}", file = volume_file)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Sep  2 11:44:08 2019

@author: ashish
"""

import pandas as pd
import numpy as np
import seaborn as sn
import matplotlib.pyplot as plt
import time

dataset = pd.read_csv('new_appdata10.csv')

# Data Preprocessing

response = dataset["enrolled"]
dataset = dataset.drop(columns='enrolled')

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(dataset, response, test_size=0.2, random_state=0)

train_identifier = X_train['user']
X_train = X_train.drop(columns='user')

test_identifier = X_test['user']
X_test = X_test.drop(columns='user')

from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train2 = pd.DataFrame(sc_X.fit_transform(X_train))
X_test2 = pd.DataFrame(sc_X.transform(X_test))
X_train2.columns = X_train.columns.values
X_test2.columns = X_test.columns.values
X_train2.index = X_train.index.values
X_test2.index = X_test.index.values
X_train = X_train2
X_test = X_test2

# Model Building

from sklearn.linear_model import LogisticRegression
classifier = LogisticRegression(random_state=0, penalty='l1')
classifier.fit(X_train, y_train)

y_predict = classifier.predict(X_test)

from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score

cm = confusion_matrix(y_test, y_predict)
accuracy_score(y_test, y_predict)
precision_score(y_test, y_predict)
recall_score(y_test, y_predict)
f1_score(y_test, y_predict)

df_cm = pd.DataFrame(cm, index=(0, 1), columns=(0,1))
plt.figure(figsize=(10,7))
sn.set(font_scale=1.4)
sn.heatmap(df_cm, annot=True, fmt='g')
print("Test Data Accuracy: %0.4f" % accuracy_score(y_test, y_predict))

from sklearn.model_selection import cross_val_score

accuracies = cross_val_score(estimator = classifier, X=X_train, y=y_train, cv=10)

print("Logistic Accuracy: %0.3f (+/- %0.3f)" % (accuracies.mean(), accuracies.std() * 2))


"""
    file: practica1
    autor: davidpillco
"""
# Importa el codecs y json
import codecs
import json
# Lee el archivo
archivo = codecs.open("datos.txt","r")
# Lee en lineas
lineas_diccionario = archivo.readlines()
# Pasa los diccionario a lista
lineas_diccionario = [json.loads(l) for l in lineas_diccionario]

# Funcion para evaluar los goles
funcion1 =lambda x:list(x.items())[1][1] > 3
goles = list(filter(funcion1, lineas_diccionario))
# Imprimer los jugadores con los goles mayores a 3
print ("Jugadores con mayor de 3 goles")
print(list(goles))

# Evaluacion de los jugadores de Nigeria
print ("Jugadores de Nigeria")
funcion2 =lambda x:list(x.items())[0][1] == "Nigeria"
pais = list(filter(funcion2, lineas_diccionario))
print(list(pais))

# Evaluacion del valor minimo y maximo de la estatura
# Valos minimo
print ("Jugador con menor estatura")
# Creo una lista de solo las estaturas
estatura = list(map(lambda x:list(x.items())[2][1], lineas_diccionario))
# Se saca el minimo
minimo = min(estatura)
# Se compara la lista con el valor minimo
funcion3 = lambda x:list(x.items())[2][1] == minimo
# Filtra el jugador con la estatura minima
est_min = list(filter(funcion3, lineas_diccionario))
# Imprime el jugador
print(est_min)

# Valor maximo
print ("Jugador con mayor estatura")
# Creo una lista de solo las estaturas
estatura = list(map(lambda x:list(x.items())[2][1], lineas_diccionario))
# Se saca el maximo
maximo = max(estatura)
# Se compara la lista con el valor maximo
funcion4 = lambda x:list(x.items())[2][1] == maximo
# Filtra el jugador con la estatura maximo
est_max = list(filter(funcion4, lineas_diccionario))
# Imprime el jugador
print(est_max)


#!/usr/bin/env python3.8
# -*- coding: utf-8 -*-
"""
https://projecteuler.net/problem=41
We shall say that an n-digit number is pandigital if it makes use of all the digits 1 to n exactly once.
For example, 2143 is a 4-digit pandigital and is also prime.

What is the largest n-digit pandigital prime that exists?
Answer: 7652413
"""
from itertools import permutations

def is_prime(n: int) -> bool:
    for i in range(2, int(n ** 0.5) + 1):
        if n % i == 0:
            return False
    return True

def largest_pandigital_prime() -> int:
    pandigital_primes = (
        number
        for length in (7, 4)
        # all other length pandigital numbers are divisible by 3 {2: 3, 3: 6, 4: 10, 5: 15, 6: 21, 7: 28, 8: 36, 9: 45}
        for number in (int(''.join(digits)) for digits in permutations(reversed('123456789'[:length]), length))
        if is_prime(number)
    )
    return next(pandigital_primes)

if __name__ == '__main__':
    from .evaluate import Watchdog

    with Watchdog() as wd:
        results = wd.evaluate(largest_pandigital_prime)(answer=7652413)


import os
import ctypes

dir_path = os.path.dirname(os.path.realpath(__file__))
lib = ctypes.cdll.LoadLibrary(dir_path  + "/" + "larcv2_to_larcv3.so")
class larcv2_to_larcv3(object):
    def __init__(self):
        lib.larcv2_to_larcv3_new.argtypes = []
        lib.larcv2_to_larcv3_new.restype = ctypes.c_void_p
        lib.larcv2_to_larcv3_add_in_file.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
        lib.larcv2_to_larcv3_add_in_file.restype = ctypes.c_void_p
        lib.larcv2_to_larcv3_set_out_file.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
        lib.larcv2_to_larcv3_set_out_file.restype = ctypes.c_void_p
        lib.larcv2_to_larcv3_initialize.argtypes = [ctypes.c_void_p]
        lib.larcv2_to_larcv3_initialize.restype = ctypes.c_void_p
        lib.larcv2_to_larcv3_convert.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
        lib.larcv2_to_larcv3_convert.restype = ctypes.c_void_p
        self.obj = lib.larcv2_to_larcv3_new()

    def add_in_file(self, in_file):
        # in_file = in_file.encode('utf-8')
        # # in_file = bytes(in_file, encoding="ascii")
        # print(in_file)
        # print(type(in_file))
        lib.larcv2_to_larcv3_add_in_file(self.obj, in_file.encode('utf-8'))

    def set_out_file(self, out_file):
        lib.larcv2_to_larcv3_set_out_file(self.obj, out_file.encode('utf-8'))

    def initialize(self):
        lib.larcv2_to_larcv3_initialize(self.obj)

    def convert(self, n_events, n_skip):
        lib.larcv2_to_larcv3_convert(self.obj, n_events, n_skip)

import argparse

def main():
    parser = argparse.ArgumentParser(description='LArCV2 to larcv3 Conversion script')

    parser.add_argument('-il','--input-larcv',required=True,
                        dest='larcv_fin',nargs='+',
                        help='string or list, Input larcv file name[s] (Required)')

    parser.add_argument('-nevents','--num-events',
                        type=int, dest='nevents', default=-1,
                        help='integer, Number of events to process')

    parser.add_argument('-nskip','--num-skip',
                        type=int, dest='nskip', default=0,
                        help='integer, Number of events to skip before processing')

    parser.add_argument('-ol','--output-larcv',default='',
                        type=str, dest='larcv_fout',
                        help='string,  Output larcv file name (optional)')

    args = parser.parse_args()
    print(args)
    converter = larcv2_to_larcv3()

    for file_name in args.larcv_fin:
        converter.add_in_file(file_name)

    if args.larcv_fout == "":
        print("No output file specified, using basename and same path as first input.")
        args.larcv_fout = args.larcv_fin[0].replace(".root", ".h5")

    converter.set_out_file(args.larcv_fout)

    converter.initialize()
    converter.convert(args.nevents, args.nskip)

if __name__ == "__main__":
    main()

# # help(conversion_libs)
# # print(conversion_libs)

# # conversion_libs.add_in_file


import typing as t

"""
When inserting a string into a trie we first check if the root node has the
first letter of the string we want to insert.

--> When inserting a new word into the Trie:
We start from the root of the tree. Then we iterate over all chars in the str
to insert. For each char (iteration), I check the current's children dict to see
if it stores the current char.
    If it does - all g, move the current there and continue char iteration
    If it doesn't - create a new node for the current char, and add to the
    current's children
Once we're done inserting the string, mark current as end, so we know there is
a str in the trie spanning from the root down to the current node.

--> When searching for a string in the Trie:
Start from the root. Iterate through the chars in the string comparing curr
char to curr node you're standing on. If at some point the char doesnt exist
in the curr nodes children --> return False

--> When deleting a string from the Trie:

"""

class TrieNode:
    def __init__(self):
        self.children: t.MutableMapping[str, TrieNode] = {}
        self._end_of_string = False

    @property
    def is_end_of_string(self) -> bool:
        return self._end_of_string

    def mark_as_end(self) -> None:
        self._end_of_string = True

    def unmark_as_end(self) -> None:
        self._end_of_string = False

class Trie:
    def __init__(self):
        self.root = TrieNode()

    def insert_string(self, string: str) -> bool:
        if not string:
            return False

        current = self.root
        for char in string:
            node = current.children.get(char)
            if not node:
                node = TrieNode()
                current.children.update({char: node})
            current = node
        current.mark_as_end()
        return True

    def search_string(self, string: str) -> bool:
        if not string:
            return True

        current = self.root
        for char in string:
            node = current.children.get(char)
            if not node:
                return False
            current = node

        return True if current.is_end_of_string else False

    @staticmethod
    def delete_string(root: TrieNode, string: str, index: int = 0) -> bool:
        char = string[index]
        current = root.children.get(char)

        if len(current.children) > 1:
            Trie.delete_string(current, string, index + 1)
            return False

        # At the last node of the string we want to delete while this string
        # is a prefix of another string. If the last node has references to
        # other characters, unmark it as end; Else delete the node entirely
        if index == len(string) - 1:
            if len(current.children) >= 1:
                current.unmark_as_end()
                return False
            else:
                root.children.pop(char)
                return True

        # Some other string is a prefix of the current string we're deleting
        if current.is_end_of_string:
            Trie.delete_string(current, string, index + 1)
            return False

        # Check if someone is dependent on the char we want to delete
        is_safe_to_delete = Trie.delete_string(current, string, index + 1)
        if is_safe_to_delete:
            root.children.pop(char)
            return True
        else:
            return False

def main():
    trie = Trie()
    print("Inserted apple:", trie.insert_string("apple"))
    print("Inserted applause:", trie.insert_string("applause"))

    print("\nChecking if apply exists:", trie.search_string("apple"))
    print("Checking if app exists:", trie.search_string("app"))

    print("\nDeleting apple")
    Trie.delete_string(trie.root, "apple")
    print("Checking if apple exists:", trie.search_string("apple"))

if __name__ == "__main__":
    main()


import argparse
import logging
import os
import shutil
import subprocess
import time

logging.basicConfig(level=logging.DEBUG)

parser = argparse.ArgumentParser(description="Famous Submitter")
parser.add_argument("-t", "--tag", type=str, help="Dataset tag.")
parser.add_argument("-i", "--input", type=str, help="Input filelist")
parser.add_argument(
    "-r",
    "--resubmits",
    type=int,
    default=10,
    help="Number of resubmissions.",
    required=False,
)
parser.add_argument(
    "-hours",
    "--hours",
    type=float,
    default=1.0,
    help="Number of hours per resubmission, in addition to the time between sample submissions.",
    required=False,
)
parser.add_argument(
    "-ms",
    "--movesample",
    type=int,
    default=0,
    help="Move each sample after submitting it (accomplishes it during the buffer time between samples set by default in monitor.py).",
)
parser.add_argument(
    "-m",
    "--move",
    type=int,
    default=0,
    help="Move all samples after all submissions (during the buffer specified by -hours).",
)
parser.add_argument(
    "-dry", "--dryrun", type=int, default=0, help="running without submission"
)

options = parser.parse_args()
nResubmits = options.resubmits
nHours = options.hours
tag = options.tag

username = os.environ["USER"]
dataDir = f"/data/submit/{username}/SUEP/{tag}/"
moveDir = f"/work/submit/{username}/SUEP/{tag}/"

# Making sure that the proxy is good
proxy_base = f"x509up_u{os.getuid()}"
home_base = os.environ["HOME"]
proxy_copy = os.path.join(home_base, proxy_base)
regenerate_proxy = False
if not os.path.isfile(proxy_copy):
    logging.warning("--- proxy file does not exist")
    regenerate_proxy = True
else:
    lifetime = subprocess.check_output(
        ["voms-proxy-info", "--file", proxy_copy, "--timeleft"]
    )
    lifetime = float(lifetime)
    lifetime = lifetime / (60 * 60)
    logging.info(f"--- proxy lifetime is {round(lifetime, 1)} hours")
    if lifetime < nHours * nResubmits * 1.5:
        logging.warning("--- proxy has expired !")
        regenerate_proxy = True

if regenerate_proxy:
    redone_proxy = False
    while not redone_proxy:
        status = os.system(
            "voms-proxy-init -voms cms --hours=" + str(nHours * nResubmits * 1.5)
        )
        if os.WEXITSTATUS(status) == 0:
            redone_proxy = True
    shutil.copyfile("/tmp/" + proxy_base, proxy_copy)

logging.info("Running resubmission script from " + str(os.environ["HOSTNAME"]))
for i in range(nResubmits):
    logging.info("Resubmission " + str(i))
    logging.info("Removing all jobs...")
    os.system("condor_rm {}".format(os.environ["USER"]))
    logging.info("Checking for corrupted files and removing them...")

    t_start = time.time()

    # delete files that are corrupted
    subDirs = os.listdir(dataDir)
    for subDir in subDirs:
        for file in os.listdir(dataDir + subDir):
            size = os.path.getsize(dataDir + subDir + "/" + file)
            if size == 0:
                subprocess.run(["rm", dataDir + subDir + "/" + file])
            elif size < 5000:
                subprocess.run(["rm", dataDir + subDir + "/" + file])

    if not options.dryrun:
        logging.info("Executing monitor.py ...")
        os.system(
            "python3 monitor.py --tag={} --input={} -r=1 -m={}".format(
                tag, options.input, options.movesample
            )
        )

    if options.move:
        if not os.path.isdir(moveDir):
            os.system("mkdir " + moveDir)

        subDirs = os.listdir(dataDir)

        for subDir in subDirs:
            if not os.path.isdir(moveDir + subDir):
                os.system("mkdir " + moveDir + subDir)

            # get list of files already in /work
            movedFiles = os.listdir(moveDir + subDir)

            # get list of files in T3
            allFiles = os.listdir(dataDir + subDir)

            # get list of files missing from /work that are in T3
            filesToMove = list(set(allFiles) - set(movedFiles))

            # move those files
            logging.info(
                "Moving " + str(len(filesToMove)) + " files to " + moveDir + subDir
            )
            for file in filesToMove:
                subprocess.run(
                    [
                        "xrdcp",
                        "root://submit50.mit.edu/" + dataDir + subDir + "/" + file,
                        moveDir + subDir + "/",
                    ]
                )

    t_end = time.time()

    # don't wait if it's the last submission
    if i == nResubmits - 1:
        logging.info("All done")
        break

    # wait to resubmit jobs using the parameter <hours>, accounts for time it took to submit them
    sleepTime = 60 * 60 * nHours
    mod = t_end - t_start
    logging.info("Submitting and moving files took " + str(round(mod)) + " seconds")
    if sleepTime - mod <= 0:
        continue
    if nHours > 0:
        logging.info("Sleeping for " + str(round(sleepTime - mod)) + " seconds")
        logging.info("(" + str(round(nHours - mod * 1.0 / 3600, 2)) + " hours)...")
    time.sleep(sleepTime - mod)


# Author: Xinyi Wang
# Date: 2021/10/05

import scipy.io as io
import numpy as np

def getCsf(mat_file, video_num):

    video_num = int(video_num)
    X_mat = io.loadmat(mat_file)
    X3 = np.asarray(X_mat['feats_mat'], dtype=np.float)
    # print(X3)
    # print(type(X3))
    cnnsa_feats = X3[video_num]
    io.savemat('../../tmp/tempmat_path/cnnsa_feats.mat', {'cnnsa_feats': cnnsa_feats})

    return cnnsa_feats

if __name__ == "__main__":
    mat_file = '/mnt/storage/home/um20242/scratch/RAPIQUE-VSFA-Saliency/feat_file/YOUTUBE_UGC_360P_RAPIQUE_VSFACNN_SALIENCY_feats.mat'
    cnnsa_feats = getCsf(mat_file, 1)
    print(cnnsa_feats)


import autograd.numpy as np
import numpy.testing as np_testing
import pytest

import pymanopt
from pymanopt.manifolds import Euclidean, Grassmann, Product, Sphere

class TestProductManifold:
    @pytest.fixture(autouse=True)
    def setup(self):
        self.m = m = 100
        self.n = n = 50
        self.euclidean = Euclidean(m, n)
        self.sphere = Sphere(n)
        self.manifold = Product([self.euclidean, self.sphere])

        point = self.manifold.random_point()

        @pymanopt.function.autograd(self.manifold)
        def cost(*x):
            return np.sum(
                [np.linalg.norm(a - b) ** 2 for a, b in zip(x, point)]
            )

        self.cost = cost

    def test_dim(self):
        np_testing.assert_equal(
            self.manifold.dim, self.m * self.n + self.n - 1
        )

    def test_typical_dist(self):
        np_testing.assert_equal(
            self.manifold.typical_dist, np.sqrt((self.m * self.n) + np.pi**2)
        )

    def test_dist(self):
        X = self.manifold.random_point()
        Y = self.manifold.random_point()
        np_testing.assert_equal(
            self.manifold.dist(X, Y),
            np.sqrt(
                self.euclidean.dist(X[0], Y[0]) ** 2
                + self.sphere.dist(X[1], Y[1]) ** 2
            ),
        )

    def test_tangent_vector_multiplication(self):
        # Regression test for https://github.com/pymanopt/pymanopt/issues/49.
        manifold = Product((Euclidean(12), Grassmann(12, 3)))
        x = manifold.random_point()
        eta = manifold.random_tangent_vector(x)
        factor = 42
        eta_scaled = eta * factor
        eta_euclidean, eta_grassmann = eta
        eta_euclidean_scaled, eta_grassmann_scaled = eta_scaled
        assert np.allclose(eta_euclidean * factor, eta_euclidean_scaled)
        assert np.allclose(eta_grassmann * factor, eta_grassmann_scaled)

    # def test_inner_product(self):

    # def test_projection(self):

    # def test_euclidean_to_riemannian_hessian(self):

    # def test_retraction(self):

    # def test_norm(self):

    # def test_random_point(self):

    # def test_random_tangent_vector(self):

    # def test_transport(self):

    def test_exp_log_inverse(self):
        s = self.manifold
        X = s.random_point()
        Y = s.random_point()
        Yexplog = s.exp(X, tangent_vector=s.log(X, Y))
        np_testing.assert_almost_equal(s.dist(point_a=Y, point_b=Yexplog), 0)

    def test_log_exp_inverse(self):
        s = self.manifold
        X = s.random_point()
        U = s.random_tangent_vector(X)
        Ulogexp = s.log(X, s.exp(X, U))
        np_testing.assert_array_almost_equal(U[0], Ulogexp[0])
        np_testing.assert_array_almost_equal(U[1], Ulogexp[1])

    def test_pair_mean(self):
        s = self.manifold
        X = s.random_point()
        Y = s.random_point()
        Z = s.pair_mean(X, Y)
        np_testing.assert_array_almost_equal(s.dist(X, Z), s.dist(Y, Z))


import json
import os

from tornado import web

from .base import BaseApiHandler, check_xsrf, check_notebook_dir
from ...api import MissingEntry

class StatusHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    def get(self):
        self.write({"status": True})

class GradeCollectionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self):
        submission_id = self.get_argument("submission_id")
        try:
            notebook = self.gradebook.find_submission_notebook_by_id(submission_id)
        except MissingEntry:
            raise web.HTTPError(404)
        self.write(json.dumps([g.to_dict() for g in notebook.grades]))

class CommentCollectionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self):
        submission_id = self.get_argument("submission_id")
        try:
            notebook = self.gradebook.find_submission_notebook_by_id(submission_id)
        except MissingEntry:
            raise web.HTTPError(404)
        self.write(json.dumps([c.to_dict() for c in notebook.comments]))

class GradeHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, grade_id):
        try:
            grade = self.gradebook.find_grade_by_id(grade_id)
        except MissingEntry:
            raise web.HTTPError(404)
        self.write(json.dumps(grade.to_dict()))

    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def put(self, grade_id):
        try:
            grade = self.gradebook.find_grade_by_id(grade_id)
        except MissingEntry:
            raise web.HTTPError(404)

        data = self.get_json_body()
        grade.manual_score = data.get("manual_score", None)
        grade.extra_credit = data.get("extra_credit", None)
        if grade.manual_score is None and grade.auto_score is None:
            grade.needs_manual_grade = True
        else:
            grade.needs_manual_grade = False
        self.gradebook.db.commit()
        self.write(json.dumps(grade.to_dict()))

class CommentHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, grade_id):
        try:
            comment = self.gradebook.find_comment_by_id(grade_id)
        except MissingEntry:
            raise web.HTTPError(404)
        self.write(json.dumps(comment.to_dict()))

    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def put(self, grade_id):
        try:
            comment = self.gradebook.find_comment_by_id(grade_id)
        except MissingEntry:
            raise web.HTTPError(404)

        data = self.get_json_body()
        comment.manual_comment = data.get("manual_comment", None)
        self.gradebook.db.commit()
        self.write(json.dumps(comment.to_dict()))

class FlagSubmissionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, submission_id):
        try:
            submission = self.gradebook.find_submission_notebook_by_id(submission_id)
        except MissingEntry:
            raise web.HTTPError(404)

        submission.flagged = not submission.flagged
        self.gradebook.db.commit()
        self.write(json.dumps(submission.to_dict()))

class AssignmentCollectionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self):
        assignments = self.api.get_assignments()
        self.write(json.dumps(assignments))

class AssignmentHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, assignment_id):
        assignment = self.api.get_assignment(assignment_id)
        if assignment is None:
            raise web.HTTPError(404)
        self.write(json.dumps(assignment))

    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def put(self, assignment_id):
        data = self.get_json_body()
        duedate = data.get("duedate_notimezone", None)
        timezone = data.get("duedate_timezone", None)
        if duedate and timezone:
            duedate = duedate + " " + timezone
        assignment = {"duedate": duedate}
        assignment_id = assignment_id.strip()
        self.gradebook.update_or_create_assignment(assignment_id, **assignment)
        sourcedir = os.path.abspath(self.coursedir.format_path(self.coursedir.source_directory, '.', assignment_id))
        if not os.path.isdir(sourcedir):
            os.makedirs(sourcedir)
        self.write(json.dumps(self.api.get_assignment(assignment_id)))

class NotebookCollectionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, assignment_id):
        notebooks = self.api.get_notebooks(assignment_id)
        self.write(json.dumps(notebooks))

class SubmissionCollectionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, assignment_id):
        submissions = self.api.get_submissions(assignment_id)
        self.write(json.dumps(submissions))

class SubmissionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, assignment_id, student_id):
        submission = self.api.get_submission(assignment_id, student_id)
        if submission is None:
            raise web.HTTPError(404)
        self.write(json.dumps(submission))

class SubmittedNotebookCollectionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, assignment_id, notebook_id):
        submissions = self.api.get_notebook_submissions(assignment_id, notebook_id)
        self.write(json.dumps(submissions))

class StudentCollectionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self):
        students = self.api.get_students()
        self.write(json.dumps(students))

class StudentHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, student_id):
        student = self.api.get_student(student_id)
        if student is None:
            raise web.HTTPError(404)
        self.write(json.dumps(student))

    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def put(self, student_id):
        data = self.get_json_body()
        student = {
            "last_name": data.get("last_name", None),
            "first_name": data.get("first_name", None),
            "email": data.get("email", None),
        }
        student_id = student_id.strip()
        self.gradebook.update_or_create_student(student_id, **student)
        self.write(json.dumps(self.api.get_student(student_id)))

class StudentSubmissionCollectionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, student_id):
        submissions = self.api.get_student_submissions(student_id)
        self.write(json.dumps(submissions))

class StudentNotebookSubmissionCollectionHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def get(self, student_id, assignment_id):
        submissions = self.api.get_student_notebook_submissions(student_id, assignment_id)
        self.write(json.dumps(submissions))

class AssignHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, assignment_id):
        self.write(json.dumps(self.api.generate_assignment(assignment_id)))

class UnReleaseHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, assignment_id):
        self.write(json.dumps(self.api.unrelease(assignment_id)))

class ReleaseHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, assignment_id):
        self.write(json.dumps(self.api.release_assignment(assignment_id)))

class CollectHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, assignment_id):
        self.write(json.dumps(self.api.collect(assignment_id)))

class AutogradeHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, assignment_id, student_id):
        self.write(json.dumps(self.api.autograde(assignment_id, student_id)))

class GenerateAllFeedbackHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, assignment_id):
        self.write(json.dumps(self.api.generate_feedback(assignment_id)))

class ReleaseAllFeedbackHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, assignment_id):
        self.write(json.dumps(self.api.release_feedback(assignment_id)))

class GenerateFeedbackHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, assignment_id, student_id):
        self.write(json.dumps(self.api.generate_feedback(assignment_id, student_id)))

class ReleaseFeedbackHandler(BaseApiHandler):
    @web.authenticated
    @check_xsrf
    @check_notebook_dir
    def post(self, assignment_id, student_id):
        self.write(json.dumps(self.api.release_feedback(assignment_id, student_id)))

default_handlers = [
    (r"/formgrader/api/status", StatusHandler),

    (r"/formgrader/api/assignments", AssignmentCollectionHandler),
    (r"/formgrader/api/assignment/([^/]+)", AssignmentHandler),
    (r"/formgrader/api/assignment/([^/]+)/assign", AssignHandler),
    (r"/formgrader/api/assignment/([^/]+)/unrelease", UnReleaseHandler),
    (r"/formgrader/api/assignment/([^/]+)/release", ReleaseHandler),
    (r"/formgrader/api/assignment/([^/]+)/collect", CollectHandler),
    (r"/formgrader/api/assignment/([^/]+)/generate_feedback", GenerateAllFeedbackHandler),
    (r"/formgrader/api/assignment/([^/]+)/release_feedback", ReleaseAllFeedbackHandler),
    (r"/formgrader/api/assignment/([^/]+)/([^/]+)/generate_feedback", GenerateFeedbackHandler),
    (r"/formgrader/api/assignment/([^/]+)/([^/]+)/release_feedback", ReleaseFeedbackHandler),

    (r"/formgrader/api/notebooks/([^/]+)", NotebookCollectionHandler),

    (r"/formgrader/api/submissions/([^/]+)", SubmissionCollectionHandler),
    (r"/formgrader/api/submission/([^/]+)/([^/]+)", SubmissionHandler),
    (r"/formgrader/api/submission/([^/]+)/([^/]+)/autograde", AutogradeHandler),

    (r"/formgrader/api/submitted_notebooks/([^/]+)/([^/]+)", SubmittedNotebookCollectionHandler),
    (r"/formgrader/api/submitted_notebook/([^/]+)/flag", FlagSubmissionHandler),

    (r"/formgrader/api/grades", GradeCollectionHandler),
    (r"/formgrader/api/grade/([^/]+)", GradeHandler),

    (r"/formgrader/api/comments", CommentCollectionHandler),
    (r"/formgrader/api/comment/([^/]+)", CommentHandler),

    (r"/formgrader/api/students", StudentCollectionHandler),
    (r"/formgrader/api/student/([^/]+)", StudentHandler),

    (r"/formgrader/api/student_submissions/([^/]+)", StudentSubmissionCollectionHandler),
    (r"/formgrader/api/student_notebook_submissions/([^/]+)/([^/]+)", StudentNotebookSubmissionCollectionHandler),
]