# Robosample tools
import sys, os, glob
import numpy as np
import scipy
import scipy.stats

def is_odd(num):
	return num % 2 != 0
#

def printOneLiner1D(series):
	''' Print Numpy array on one line'''
	for i in range(series.shape[0]):
		print (series[i]),
	print()
#

def print2D(M):
	''' Print Numpy 2D array on row per line'''
	for i in range(M.shape[0]):
		for j in range(M.shape[1]):
			print (M[i, j]),
		print()
#

def print2DAndExit(M):
	for i in range(M.shape[0]):
		for j in range(M.shape[1]):
			print (M[i, j]),
		print()
	exit(0)
#

def moving_average(series, window_size):
	if np.size(series.shape) > 1:
		print("Moving average function error: Implemented only for 1D data.")
		return 1
	if (window_size % 2 != 1):
		window_size = window_size + 1
		print("Moving average function warning: Window size increse by 1.")

	shift = int((window_size - 1) / 2)
	#retSeries = np.ones((series.shape[0] - (2*shift)))
	retSeries = np.zeros((series.shape[0]))
	for i in range(shift, (retSeries.shape[0] - shift)):
		retSeries[i] = np.mean(series[i - shift: i + shift + 1])
	return retSeries
#

def cumulative_average(series):
	if np.size(series.shape) > 1:
		print("Cumulative average function error: Implemented only for 1D data.")
		return 1
	retSeries = np.zeros((series.shape[0]))
	retSeries[0] = series[0]
	for i in range(1, retSeries.shape[0]):
		retSeries[i] = np.mean(series[0:i])
	return retSeries
#

def difference_quotient(series, window_size):
	if np.size(series.shape) > 1:
		print("Difference quotient function error: Implemented only for 1D data.")
		return 1
	if (window_size % 2 != 1):
		window_size = window_size + 1
		print("Difference quotient function warning: Window size increse by 1.")
	shift = int((window_size - 1) / 2)
	retSeries = np.zeros((series.shape[0]))
	for i in range(shift, (retSeries.shape[0] - shift)):
		retSeries[i] = (series[i + shift] - series[i - shift]) / window_size
	return retSeries
#

def moving_difference(series, window_size):
	if np.size(series.shape) > 1:
		print("Moving difference function error: Implemented only for 1D data.")
		return 1
	if (window_size % 2 != 1):
		window_size = window_size + 1
		print("Moving difference function warning: Window size increse by 1.")
	shift = int((window_size - 1) / 2)
	retSeries = np.zeros((series.shape[0]))
	for i in range(shift, (retSeries.shape[0] - shift)):
		retSeries[i] = (series[i + shift] - series[i - shift])
	return retSeries
#

def intersections(series1, series2):
	'''Intersection of two series'''
	minSize = np.min([series1.size, series2.size])
	intersections = []
	diff = prevDiff = 0.0
	for i in range(minSize):
		if not np.isnan(series1[i]):
			diff = series1[i] - series2[i]
			if (diff * prevDiff) < 0:
				intersections.append(i)
				#break
			prevDiff = diff
	return np.array(intersections)
#

# Take the product (f(t))(f(t+lag)) and average over (n - lag) points
# Normalize. The variance is also the correlation at time 0 and
# should be highesta
def CestGrossfield(argM, series):
	# Define a zero
	tiny = 0.0000001

	# Moments
	n = series.size
	mean = np.mean(series)
	variance = np.var(series)
	#print("CestGrossfield mean var", mean, variance)

	# Vars
	lags = np.arange(0, argM)
	cut = 0
	resultF = np.zeros((argM))

	for lag in lags:

		#print("CestGrossfield s-mu", series[0 : (n - lag)] - mean)
		#print("CestGrossfield s-mu", series[lag : n]       - mean)

		resultF[lag] = np.mean( \
			(series[0 : (n - lag)] - mean) * \
			(series[lag : n]       - mean) ) \
			/ variance

		#print("CestGrossfield", lag,  resultF[lag])

		# If the correlation function reaches 0 the rest is considered noise
		if resultF[lag] < tiny:
			cut = lag
			break

	# Integrated autocorrelation time and effective sample size
	integratedAC = np.sum(resultF)
	effSampleSize = n / integratedAC

	return resultF, cut, integratedAC, effSampleSize
#

autocorrLabels = ['FFT Partial', 'FFT Nonpartial', 'NumPy Correlate Nonpartial', 'NumPy Correlate Partial', 'Manual']

# Functions from https://stackoverflow.com/users/2005415/jason
def autocorrDetail(argM, series):
	'''fft, don't pad 0s, non partial'''

	# Define a zero
	tiny = 0.0000001

	n = series.size

	if argM == 0:
		argM = n
	lags = np.arange(0, argM)
	mean = series.mean()
	var = np.var(series)
	seriesp = series - mean

	cf = np.fft.fft(seriesp)
	freq = np.fft.fftfreq(n)
	#return (freq, cf)
	sf = cf.conjugate()*cf
	#return (freq, sf)
	invSf = np.fft.ifft(sf)

	resultF = invSf.real/var/len(series)

	return (freq, cf, sf, invSf, resultF)

	#cut = 0
	#for i in range(resultF.shape[0]):
	#	if resultF[i] < tiny:
	#		cut = i
	#		break
	#resultF[cut:] = 0

	#print(resultF)
	#return resultF[:len(lags)]

	# Integrated autocorrelation time and effective sample size
	#integratedAC = np.sum(resultF)
	#effSampleSize = n / integratedAC

	#return resultF, cut, integratedAC, effSampleSize
#

# Functions from https://stackoverflow.com/users/2005415/jason
def autocorr1(argM, series):
	'''fft, don't pad 0s, non partial'''

	# Define a zero
	tiny = 0.0000001

	n = series.size

	lags = np.arange(0, argM)
	mean = series.mean()
	var = np.var(series)
	seriesp = series - mean

	cf = np.fft.fft(seriesp)
	sf = cf.conjugate()*cf
	resultF = np.fft.ifft(sf).real/var/len(series)

	cut = 0
	for i in range(resultF.shape[0]):
		if resultF[i] < tiny:
			cut = i
			break
	resultF[cut:] = 0

	#print(resultF)
	#return resultF[:len(lags)]

	# Integrated autocorrelation time and effective sample size
	integratedAC = np.sum(resultF)
	effSampleSize = n / integratedAC

	return resultF, cut, integratedAC, effSampleSize
#

def autocorr2(argM, x):
	'''fft, pad 0s, non partial'''

	lags = np.arange(0, argM)
	n=len(x)
	# pad 0s to 2n-1
	ext_size=2*n-1
	# nearest power of 2
	fsize=2**np.ceil(np.log2(ext_size)).astype('int')

	xp=x-np.mean(x)
	var=np.var(x)

	# do fft and ifft
	cf=np.fft.fft(xp,fsize)
	sf=cf.conjugate()*cf
	corr=np.fft.ifft(sf).real
	corr=corr/var/n

	return corr[:len(lags)]

def autocorr3(argM, x):
	'''numpy.corrcoef, partial'''

	lags = np.arange(0, argM)
	corr=[1. if l==0 else np.corrcoef(x[l:],x[:-l])[0][1] for l in lags]
	return np.array(corr)

def autocorr4(argM, x):
	'''numpy.correlate, non partial'''
	lags = np.arange(0, argM)
	mean=x.mean()
	var=np.var(x)
	xp=x-mean
	corr=np.correlate(xp,xp,'full')[len(x)-1:]/var/len(x)

	return corr[:len(lags)]
#

def autocorr5(argM, x):
	'''manualy compute, non partial'''

	lags = np.arange(0, argM)
	mean=np.mean(x)
	var=np.var(x)
	xp=x-mean
	corr=[1. if l==0 else np.sum(xp[l:]*xp[:-l])/len(x)/var for l in lags]

	return np.array(corr)


"""This file contains the migration mixin for SQLite.
The mixin has the same behavior as the postgresql migrator,
but has different logging messages.
Hence, the methods are overriden to account for this."""

from kinto.core.storage.postgresql import migrator
import logging
import os

logger = logging.getLogger(__name__)

class SQLiteMigratorMixin(migrator.MigratorMixin):
    """Migrator for SQLite. Subclasses postgresql
    migration mixin as the behavior is the same.
    """

    # Basically the exact same thing as PostgreSQL,
    # but it has the PostgreSQL word replaced with SQLite
    # in the logger messages.
    def create_or_migrate_schema(self, dry_run=False):
        """Either create or migrate the schema, as needed."""
        version = self.get_installed_version()
        if not version:
            self.create_schema(dry_run)
            return

        logger.info('Detected SQLite {} schema version {}.'.format(self.name, version))
        if version == self.schema_version:
            logger.info('SQLite {} schema is up-to-date.'.format(self.name))
            return

        self.migrate_schema(version, dry_run)

    # Basically the exact same thing as PostgreSQL,
    # but it has the PostgreSQL word replaced with SQLite
    # in the logger messages.
    def create_schema(self, dry_run):
        """Actually create the schema from scratch using self.schema_file.

        You can override this if you want to add additional sanity checks.
        """
        logger.info('Create SQLite {} schema at version {} from {}'.format(
            self.name, self.schema_version, self.schema_file))
        if not dry_run:
            self._execute_sql_file(self.schema_file)
            logger.info('Created SQLite {} schema (version {}).'.format(
                self.name, self.schema_version))

    # Basically the exact same thing as PostgreSQL,
    # but it has the PostgreSQL word replaced with SQLite
    # in the logger messages.
    def migrate_schema(self, start_version, dry_run):
        migrations = [(v, v + 1) for v in range(start_version, self.schema_version)]
        for migration in migrations:
            expected = migration[0]
            current = self.get_installed_version()
            error_msg = 'SQLite {} schema: Expected version {}. Found version {}.'
            if not dry_run and expected != current:
                raise AssertionError(error_msg.format(self.name, expected, current))

            logger.info('Migrate SQLite {} schema from version {} to {}.'.format(
                self.name, *migration))
            filename = 'migration_{0:03d}_{1:03d}.sql'.format(*migration)
            filepath = os.path.join(self.migrations_directory, filename)
            logger.info('Execute SQLite {} migration from {}'.format(self.name, filepath))
            if not dry_run:
                self._execute_sql_file(filepath)
        logger.info('SQLite {} schema migration {}.'.format(
            self.name,
            'simulated' if dry_run else 'done'))

    # Helper method to execute the schema file
    def _execute_sql_file(self, filepath):
        """Helper method to execute the SQL in a file."""
        print("Creating schema")
        with open(filepath) as f:
            schema = f.read()

        # Since called outside request, force commit.
        with self.client.connect() as conn:
            results = conn.executescript(schema)
            print("Schema results")
            print(results.fetchone())


#!/bin/python3

import math
import os
import random
import re
import sys

# Complete the bomberMan function below.
willExplodeAt = {}
def bomberMan(n, grid):
    # second 1 - no change - return initial state
    if n == 1: return grid
    # convert the initial locations to a dict
    rows = len(grid)
    cols = len(grid[0])
    # this dict will contain cells neighboring the initial bomb-placements
    #  the values will be:
    #   1 --> "pure" neighbors - having only initial-bomb and other in-range neighbors
    #   0 --> "mixed" neighbors - having "other" neighbors as  well
    neighbors = {}
    for i in range(rows):
        for j in range(cols):
            if grid[i][j] == 'O':
                willExplodeAt[cellName(i, j)] = 3
            else:
                willExplodeAt[cellName(i, j)] = 0
    # identify all those "In-range" of an initial bomb
    for i in range(rows):
        for j in range(cols):
            if willExplodeAt[cellName(i, j)] == 0: continue
            above = cellName(i - 1, j)
            if i > 0 and willExplodeAt[above] == 0: neighbors[above] = 1
            below = cellName(i + 1, j)
            if i < rows - 1 and willExplodeAt[below] == 0: neighbors[below] = 1
            right = cellName(i, j + 1)
            if j < cols - 1 and willExplodeAt[right] == 0: neighbors[right] = 1
            left = cellName(i, j - 1)
            if j > 0 and willExplodeAt[left] == 0: neighbors[left] = 1
    # scan the in-range set to determine whether they are neighbors with "other"
    for cell in neighbors.keys():
        (i, j) = cellLocation(cell)
        if i > 0:
            above = cellName(i - 1, j)
            if willExplodeAt[above] == 0 and above not in neighbors:
                neighbors[cell] = 0
                continue
        if i < rows - 1:
            below = cellName(i + 1, j)
            if willExplodeAt[below] == 0 and below not in neighbors:
                neighbors[cell] = 0
                continue
        if j < cols - 1:
            right = cellName(i, j + 1)
            if willExplodeAt[right] == 0 and right not in neighbors:
                neighbors[cell] = 0
                continue
        if j > 0:
            left = cellName(i, j - 1)
            if willExplodeAt[left] == 0 and left not in neighbors:
                neighbors[cell] = 0
                continue

    # direct approach
    for cell in willExplodeAt.keys():
        # seems to be 3 types of cells:
        # 1. those initially populated with a bomb
        # 2. those in-range of those above (and not adjacent to an "other")
        # 3. like 2 above, except also neighboring "others" (below)
        # 3. others (those not initially in-range of bombs)
        #
        if cell in neighbors.keys():
            # type 3 - must go 1st
            if neighbors[cell] == 0:
                willExplodeAt[cell] = 1 if n % 2 == 0 else 0
            # type 2 - next
            else:
                willExplodeAt[cell] = 0 if n % 4 == 3 else 1
            continue
        # type 1 - and these next
        if willExplodeAt[cell] > 0:
            willExplodeAt[cell] = 0 if n % 4 == 3 else 1
            continue
        # type 4
        if willExplodeAt[cell] == 0:
            # applies to cells not initially in-range of a bomb
            willExplodeAt[cell] = 0 if n % 4 == 1 else 1
    return gridState(rows,cols)

def cellName(i, j): return f'{i}+{j}'
def cellLocation(cell: str):
    indexStrings = cell.split('+')
    return [int(indexStrings[0]),int(indexStrings[1])]

def gridState(rows,cols):
    result = []
    for i in range(rows):
        row = ''
        for j in range(cols):
            row += 'O' if willExplodeAt[cellName(i,j)] > 0 else '.'
        result.append(row)
    return result

if __name__ == '__main__':
    try:
        fptr = open(os.environ['OUTPUT_PATH'], 'w')
    except KeyError:
        fptr = sys.stdout

    rcn = input().split()

    r = int(rcn[0])

    c = int(rcn[1])

    n = int(rcn[2])

    grid = []

    for _ in range(r):
        grid_item = input()
        grid.append(grid_item)

    result = bomberMan(n, grid)

    fptr.write('\n'.join(result))
    fptr.write('\n')

    fptr.close()


lst=[]
fh = open("mbox-short")
count = 0
for lines in fh:
    if not lines.startswith("From "):
        continue
    data=lines.rstrip().split()
    print(data[1])
    count+=1

    # if dat[0]!="From":
    #     continue
    # else:
    #     print(dat[1])
    #     count+=1

    # print(dat)
    # count+=1

print("There were",count, "lines in the file with From as the first word")


app = Flask(__name__)
analyzer = SentimentIntensityAnalyzer()

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/analyze', methods=['POST'])
def analyze():
    text = request.form['text']
    sentiment = analyze_sentiment(text)
    return {"sentiment": sentiment}

def analyze_sentiment(text):
    sentiment_scores = analyzer.polarity_scores(text)
    compound_score = sentiment_scores['compound']

    if compound_score >= 0.05:
        sentiment = "Positive"
    elif compound_score <= -0.05:
        sentiment = "Negative"
    else:
        sentiment = "Neutral"

    return sentiment

if __name__ == '__main__':
    app.run(debug=True)


#!/usr/bin/python3

import requests
import json
import subprocess
#
# Monitoring Upload
#

who = requests.get('http://localhost:5000/who')
os = requests.get('http://localhost:5000/os/kernel')
swap = requests.get('http://localhost:5000/swap/so')
mem = requests.get('http://localhost:5000/mem/free')
cpu = requests.get('http://localhost:5000/cpu/sy')
disk = requests.get ('http://localhost:5000/disk')
dicSwap = swap.json()
dicWho = who.json()
dicOs = os.json()
dicMem = mem.json()
dicCpu = cpu.json()
dicDisk = disk.json()
payload = {**dicWho, **dicOs, **dicSwap, **dicMem, **dicCpu, **dicDisk}
print (payload)
postdisk = requests.post('https://semana2.herokuapp.com/monitoring-post', json = payload)
print (postdisk.status_code)

#
# Transmission Magnetlinks requests
#
urls = requests.get('https://semana2.herokuapp.com/magnetlinks')
dicUrls = urls.json()
values = dicUrls.values()
for magnet in values:
    subprocess.check_output(["transmission-remote", "-a", magnet])
#
# Transmission Magnetlinks status
#

keys = ['ID', 'progress', 'downloaded', 'size' ,'ETA', 'speedup', 'speeddown', 'ratio', 'status-1', 'name']
keysA = ['ID', 'progress', 'downloaded', 'size', 'ETA', 'speedup', 'speeddown', 'ratio', 'status-1', 'status-2', 'status-3', 'name']

def list (value):

    transmission = subprocess.Popen(['transmission-remote' , '-l'],stdout = subprocess.PIPE)
    tail = subprocess.Popen(['tail', '-n' , '+2'], stdin = transmission.stdout,stdout = subprocess.PIPE)
    tr = subprocess.Popen(['tr', '-s' , ' '], stdin = tail.stdout, stdout = subprocess.PIPE)

    output = subprocess.check_output(['cut' , '-d', '\n', '-f', str(value)], stdin = tr.stdout).decode('utf-8').strip()
    return output

def generate():
    x = 1
    payload = []
    while True:
        aux = list(x)
        if ((aux.find("Sum:")) != -1)  :
            break
        else:
            x += 1

        data = {}
        if ((aux.find("up & down")) == -1):
            values = aux.split(" ")
            for i in range (0 , 10):
                data[keys[i]] = values[i]
        else:
            for j in range (0, 12):
                data[keysA[j]] = values[j]
        payload.append(data)
    print (payload)
    return payload

post = requests.post('http://localhost:5001/status', json = generate())
print (post.status_cod)


qtd_testes = int(input())

for _ in range(qtd_testes):
    blocos = []
    nl, nc, x, y = [int(x) for x in input().split()]
    for z in range(nl):
        blocos.append([int(x) for x in input().split()])

    for i in range(nl):
        for j in range(nc):
            distancia_x = 10-abs((x-1-i)) if 10-abs((x-1-i)) >= 1 else 1
            distancia_y = 10-abs((y-1-j)) if 10-abs((y-1-j)) >= 1 else 1
            blocos[i][j] += min(distancia_x,distancia_y)
    print(f"Parede {_+1}:")
    for z in range(nl):
        print(*blocos[z])

Module 7 - Bidimensional Array (Matrix)/soco_hulk.py


from pprint import pprint

import os

import googleapiclient.discovery
from googleapiclient.discovery import build
import subprocess
import requests

import youtube_dl
from yt_dlp import YoutubeDL
from youtube_dl.utils import DateRange
import time
from dotenv import load_dotenv
from pathlib import Path

def get_video_list(key, term, maxVals=20, live=False, order_v = "relevance"):
    # ================================================
    # calling the API
    api_service_name = "youtube"
    api_version = "v3"

    youtube = googleapiclient.discovery.build(api_service_name, api_version, developerKey = key)

    if live == True:
        request = youtube.search().list(
            part="id",
            channelType="any",
            eventType="live",
            maxResults=maxVals,
            order=order_v,
            q=term,
            safeSearch="none",
            type="video",
            videoCaption="any"
        )
    if live == False:
        request = youtube.search().list(
            part="id",
            channelType="any",
            maxResults=maxVals,
            order=order_v,
            q=term,
            safeSearch="none",
            type="video",
            videoCaption="any"
        )
    response = request.execute()
    return response

def get_video_list_before_year(key, term, date, maxVals=20, live=False, order_v = "relevance"):
    # ================================================
    # calling the API
    api_service_name = "youtube"
    api_version = "v3"

    youtube = googleapiclient.discovery.build(api_service_name, api_version, developerKey = key)
    request = youtube.search().list(
        part="id",
        channelType="any",
        maxResults=maxVals,
        order=order_v,
        q=term,
        safeSearch="none",
        type="video",
        videoCaption="any",
        publishedBefore=date+'-01-01T00:00:00Z'
    )
    response = request.execute()
    return response

def get_title(id1, key):
	youtube = build('youtube', 'v3', developerKey=key)
	print("id1 is: ", id1)
	url = id1
	stripped_url = url.replace("https://www.youtube.com/watch?v=", '')
	print(stripped_url)
	title = ""
	results = youtube.videos().list(id=stripped_url, part='snippet').execute()
	print("the result is: ", results)
	for result in results.get('items', []):
		print("getting the title ")
		print(result['id'])
		print(result['snippet']['title'])
		title = result['snippet']['title']
	return title

def get_ids(resp):
    # ================================================
    # Getting the youtube IDs out of the API response

    items = resp["items"]

    youtube_ids = []
    for item in items:
        yt_id = item["id"]
        youtube_ids.append(yt_id['videoId'])

    print(youtube_ids)
    print("the first one is: ")
    print(youtube_ids[0])
    print(len(youtube_ids))

    url_basic = "https://www.youtube.com/watch?v="
    full_youtube_ids = []
    val = 0
    for x in youtube_ids:
        full_youtube_ids.append( url_basic + x )
        val=val+1
    print("full ids are: ", full_youtube_ids)
    return full_youtube_ids

def start_up_stream(ids, id_number):
	# ================================================
	# open video stream
	# ================================================
	command = 'timeout 15s streamlink -p "omxplayer --orientation 180" --player-fifo '

	command = command + ids[id_number] + ' best'
	print("the command is " + command)
	os.system(command)

def start_up_stream_and_close_stream(ids):
	# ================================================
	# open video stream
	# ================================================
	command = 'streamlink -p "omxplayer --timeout 20" --player-fifo '

	command = command + ids[id_val] + ' best'
	os.system(command)

def remove_unwanted(titles, ids, words_to_avaoid):
    # ================================================
    # removing unwanted titles
    # ================================================
    ids_size = len( the_ids)
    id_val = ids_size - 1
    for word in words_to_avaoid:
        if word in titles:
            # remove from list
            print("we need to remove this!")
            ids.remove(ids[id_val]) # check this
            ids_size = len(ids)
            print("the second id_size is: ", ids_size)
            id_val = ids_size - 1
    return ids, ids_size

def download_videos_simple(url, output_loc, max="100M"):
    Path(output_loc).mkdir(parents=True, exist_ok=True)
    out = output_loc + "%(title)s-%(id)s.%(ext)s"
    ydl_opts = {
    'format': 'best',
    'max': '100M',
    'outtmpl': out,
    }
    with YoutubeDL(ydl_opts) as ydl:
        ydl.download([url])
    # subprocess.run(['youtube-dl', '-o', output_loc, '-f', 'best', '--max-filesize', max, url], capture_output=True)
    print("video downloaded")

def download_videos_during_period(url, output_loc, before, after, max="100M"):
    Path(output_loc).mkdir(parents=True, exist_ok=True)
    out = output_loc + "%(title)s-%(id)s.%(ext)s"
    ydl_opts = {
    'format': 'best',
    'max': '100M',
    'outtmpl': out,
    'daterange':DateRange(after, before),
    }
    with YoutubeDL(ydl_opts) as ydl:
        ydl.download([url])
    # subprocess.run(['youtube-dl', '-o', output_loc, '-f', 'best', '--max-filesize', max, '-dateafter', after, '-datebefore', before, url], capture_output=True)
    print("downloaded videos in playlist")

def download_videos_before(url, output_loc, before, max="100M"):
    subprocess.run(['youtube-dl', '-o', output_loc, '-f', 'best', '--max-filesize', max, '-datebefore', before, url], capture_output=True)
    print("downloaded videos in playlist")

def download_videos_in_list(list_v, output_loc, max="100M"):
    for video in list_v:
        download_videos_simple(video, output_loc, max)
    print("all videos downloaded")

def save_ids(ids, output_loc, filename):
    with open(output_loc + filename + '.txt', 'w') as txt_file:
        for line in ids:
            line_str = [str(n) for n in line] # create string list from integer list
            txt_file.write(" ".join(line_str) + "\n")
    print("file saved at: ", output_loc + filename + ".txt")

def load_urls_from_file(filepath):
    with open(filepath) as file:
        lines = [line.rstrip() for line in file]
    print("the array is: ", lines)
    return lines

def set_up_environment():
    load_dotenv()
    key = os.environ.get("YOUTUBE_DATA_API_KEY") # Define the API Key. This should be in a hidden file
    return key

# =========================================================================
# Main Loop
if __name__ == "__main__":
    id_val = 0
    # avoid_these = ["Meme", "meme", "MEME","Coffin","COFFIN", "Coffin", "Dance", "DANCE", "dance", "Cleanse", "CLEANSE", "cleanse", "Vibration", "VIBRATION", "vibration", "Sonic", "SONIC", "sonic", "Lightning", "LIGHTNING", "lightning", "McQueen","GTA", "gta", "Gta", "Minecraft", "MINECRAFT", "Memes", "MEMES", "memes", "Funny", "FUNNY", "funny", "Birds", "BIRDS", "birds", "TV", "tv", "Tv"]
    # load_dotenv()
    # DEVELOPER_KEY  = os.environ.get("YOUTUBE_DATA_API_KEY") # Define the API Key. This should be in a hidden file
    DEVELOPER_KEY = set_up_environment()
    api_response = get_video_list(DEVELOPER_KEY, "another walk", False, "relevance") #final term could be rating, title, video count, date
    print(api_response)

    the_ids = get_ids(api_response)
    print("the ids are: ", the_ids)

    print("the number of ids is: ", ids_size)
    counter = 0
    current_title = get_title(the_ids[id_val])
    print("the current title is: ", current_title)

    the_ids, id_size = remove_unwanted(current_title, the_ids, avoid_these)

# download videos with youtube_dl

# start_up_stream(the_ids, id_val)
# start = time.time()
# start_callapi = time.time()

# # ================================================
# # main loop
# # ================================================
# while True:
# 	# everything in here will loop until told not to
# 	try:
# 		# when some timer is up startup another subprocess called process 2 and then kill the process1
# 		if(time.time()-start>12.5):
# 			if(id_val<0):
# 				id_val=ids_size
# 			else:
# 				id_val = id_val - 1
# 			# add a check here for unwanted titles
# 			start_up_stream(the_ids, id_val)
# 			time.sleep(5)
# 			start_up_stream(the_ids,id_val)
# 			print("the youtube link is; ", the_ids[id_val])
# 			print("starting up the stream")
# 			start = time.time()

# 		if(time.time()-start_callapi>360):
# 			# 360 is 6 minutes 6*60
# 			# change this value to be how often you want to refresh the streams
# 			# this will happen when the code starts up
# 			api_response = call_api()
# 			print(api_response)
# 			the_ids = get_ids(api_response)
# 			ids_size = len( the_ids)
# 			print("new set of ids")
# 			print(the_ids)
# 			start_callapi = time.time()

# 	except KeyboardInterrupt:
# 		print('Interrupted')
# 		try:
# 			sys.exit(0)
# 		except SystemExit:
# 			os._exit(0)


from datetime import datetime, timedelta, timezone

import pinhook.plugin

def nick_key(message):
    return "{}.{}".format(message.channel, message.nick)

def greetings_key(message):
    return "greetings.messages.{}".format(nick_key(message))

@pinhook.plugin.listener("greetings")
def greetings_plugin(message):
    greetings_interval = timedelta(minutes=30)

    last_seen_key = "greetings.last_seen.{}".format(nick_key(message))
    channel_greetings_key = "greetings.messages.{}".format(message.channel)

    now = datetime.now(timezone.utc)

    last_seen = message.bot.datastore.getset(last_seen_key, now.isoformat())
    if last_seen is not None:
        last_seen = datetime.fromisoformat(last_seen)
    else:
        last_seen = datetime.now(timezone.utc) - timedelta(days=1)

    if (now - last_seen) < greetings_interval:
        # Too soon to say anything
        return

    greeting = message.bot.datastore.srandmember(greetings_key(message))
    if greeting is None:
        greeting = message.bot.datastore.srandmember(channel_greetings_key)

    if greeting is not None:
        return pinhook.plugin.message(greeting)

@pinhook.plugin.command('~add-greeting', help_text='Add a personalized greeting.')
def add_greeting_plugin(message):
    if datetime.now(timezone.utc) - message.bot.get_join_time(message.channel) > timedelta(seconds=15):
        message.bot.datastore.sadd(greetings_key(message), message.arg)
        return pinhook.plugin.action("whirrs and buzzes as he records the new greeting")


import os
import shutil
import unittest

from geojson_modelica_translator.scaffold import Scaffold

class ScaffoldTest(unittest.TestCase):
    def setUp(self):
        self.data_dir = os.path.join(os.path.dirname(__file__), 'data')
        self.output_dir = os.path.join(os.path.dirname(__file__), 'output')
        if os.path.exists(self.output_dir):
            shutil.rmtree(self.output_dir)
        os.makedirs(self.output_dir)

    def test_scaffold(self):
        scaffold = Scaffold(self.output_dir, "scaffold_01", overwrite=True)
        scaffold.create()
        self.assertTrue(
            os.path.exists(os.path.join(self.output_dir, "scaffold_01", "Resources", "Scripts", "Loads", "Dymola"))
        )

    # def test_add_building(self):
    #     scaffold = Scaffold(self.output_dir, "scaffold_02", overwrite=True)
    #     load_1 = FakeConnector(None)
    #     self.assertIsInstance(load_1, building_base)
    #     scaffold.loads.append(load_1)
    #     scaffold.create()
    #
    #     r = scaffold.to_modelica()
    #     self.assertTrue(r)


import unittest

from bankline_parser.data_services import enums, fields
from bankline_parser.data_services.exceptions import ParseError

class DataServicesFieldsTestCase(unittest.TestCase):
    def test_text_field(self):
        row = 'amazing'
        text_field = fields.TextField(0, 7)
        self.assertEqual(text_field.parse(row), 'amazing')

    def test_text_field_with_padding(self):
        row = '   amazing'
        text_field = fields.TextField(0, 10)
        self.assertEqual(text_field.parse(row), 'amazing')

    def test_text_field_with_padding_left_justified(self):
        row = 'amazing   '
        text_field = fields.TextField(0, 10, justification='l')
        self.assertEqual(text_field.parse(row), 'amazing')

    def test_text_field_left_justified_left_padding_not_removed(self):
        row = '   amazing'
        text_field = fields.TextField(0, 10, justification='l')
        self.assertEqual(text_field.parse(row), '   amazing')

    def test_numeric_field(self):
        row = 'otherstuff000064'
        numeric_field = fields.NumericField(10, 16)
        self.assertEqual(numeric_field.parse(row), 64)

    def test_numeric_field_nonnumeric_value_fails(self):
        row = 'h67'
        numeric_field = fields.NumericField(0, 3)
        try:
            numeric_field.parse(row)
            self.fail()
        except ParseError:
            pass

    def test_enum_field(self):
        row = 'gggggD000000'
        enum_field = fields.EnumField(5, 6, enums.BalanceType)
        self.assertEqual(enum_field.parse(row), enums.BalanceType.debit)

    def test_static_field(self):
        row = 'hello'
        static_field = fields.StaticField(0, 6, 'hello')
        self.assertEqual(static_field.parse(row), 'hello')

    def test_static_field_trim_padding(self):
        row = '   hello'
        static_field = fields.StaticField(0, 9, 'hello')
        self.assertEqual(static_field.parse(row), 'hello')

    def test_static_field_include_padding(self):
        row = '   hello'
        static_field = fields.StaticField(0, 9, '   hello', pad_char=None)
        self.assertEqual(static_field.parse(row), '   hello')

    def test_static_field_incorrect_value_fails(self):
        row = 'herro'
        static_field = fields.StaticField(0, 6, 'hello')
        try:
            static_field.parse(row)
            self.fail()
        except ParseError:
            pass


from datetime import datetime
import json
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import os

WINDOW = "1920,1080"

chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--window-size=%s' %WINDOW)

headless_browsers = []
chrome_instances = 5

for i in range(chrome_instances):
    driver= webdriver.Chrome(chrome_options=chrome_options)
    headless_browsers.append(driver)

class ImageDownloader:
    """
        ImageDownloader class will create chrome brower instances.
        It will load the webpages from the urls list and save a ss in temp folder
    """
    def __init__(self):
        self.urlsMapper = {}
        self.count = 0
        self.mapper = {}

    def addUrl(self, url):
        filename =  datetime.now().isoformat()+".png"
        self.mapper[filename] = False
        self.urlsMapper[filename] = url
        res = self.getImage(filename)
        if(res == "error"):
            return res
        del self.mapper[filename]
        del self.urlsMapper[filename]
        return "/static/temp/"+filename

    def getImage(self, filename):
        driver = headless_browsers[self.count % chrome_instances]
        self.count+=1
        current_url = self.urlsMapper[filename]
        if not current_url.startswith('http'):
            return 'error'
        driver.get(current_url)
        driver.save_screenshot(os.path.join('static','temp',filename))
        self.mapper[filename] = True
        return True

    def removeOldFiles(self):
        currentTimeStamp = datetime.now().timestamp()
        with os.scandir('static/temp') as s:
            for file in s:
                lastModified = os.path.getmtime(file)
                diff = currentTimeStamp - lastModified
                if((diff/(24*60*60)) > 1):
                    os.remove(file)
                    print(file)

    def getCount(self):
        return self.count

if __name__=="__main__":
    i = ImageDownloader()
    i.removeOldFiles()


import unittest
import aspose.words as aw
import os
import platform

# root_dir =  "X:/" if platform.system() == "Windows" else "/X/"
root_dir = os.path.abspath(os.curdir) + "/"
root_dir = root_dir[:root_dir.find("Aspose.Words-for-Python-via-.NET")]
api_examples_root = root_dir + "Aspose.Words-for-Python-via-.NET/Examples/"
license_path = root_dir + "Aspose.Words-for-Python-via-.NET/Temp/Aspose.Words.Python.NET.lic"
my_dir = api_examples_root + "Data/"
artifacts_dir = my_dir + "Artifacts/"
golds_dir = my_dir + "Golds/"
temp_dir = my_dir + "Temp/"
image_dir = my_dir + "Images/"
fonts_dir = my_dir + "MyFonts/"
aspose_logo_url = "https://www.aspose.cloud/templates/aspose/App_Themes/V3/images/words/header/aspose_words-for-net.png"

class ApiExampleBase(unittest.TestCase):

    def setUp(self):
        if os.path.exists(artifacts_dir):
            l = aw.License()
            l.set_license(license_path)
        if not os.path.exists(artifacts_dir):
            os.makedirs(artifacts_dir)


"""error"""

from typing import Callable
import sys

def input_error(func: Callable[..., None]) -> Callable[..., None]:
    """Decorator for handling input errors"""
    def wrapper_input_error(*args: tuple, **kwargs: dict) -> None:
        """Wrapper function for handling input errors"""
        try:
            func(*args, **kwargs)

        except TypeError as error:
            print(f"TypeError: {error}")
            sys.exit('Try again!')

        except ValueError as error:
            print(f"ValueError: {error}")
            sys.exit('Try again!')

        except KeyError as error:
            print(f"KeyError: {error}")
            sys.exit('Try again!')

    return wrapper_input_error


# -*- coding: utf-8 -*-
### The working functions

### Joseph Hostyk

### 2018-4-9

from helper import *

###################################################################################################

### First steps: compare our results to GTR's.

###################################################################################################

def getKeyTerms():

	GTRfilesPath = PROJECT + "Results/GeneSets/GTRgeneLists/"

	filenames = os.listdir(GTRfilesPath)
	keyTerms = [filename.replace(".tsv", "") for filename in filenames if ".tsv" in filename]

	print ("We have {} keyterms.".format(len(keyTerms)))
	return keyTerms

def createOurGeneSets(keyWords, outputFolder):

	allSynopses = getAllSynopses()
	officialGeneList = getOfficialGenes()
	# inheritanceDictionary = makeDictFromTwoColumns(PROJECT + "allGenes/{}/allGenesInheritances.txt".format(CURRENT_DATE_VERSION), "\t")

	print ("Parsing synopses...")
	phenoMimsToGenesDict = makeMIMdict()
	geneSets = defaultdict(set)

	for _, synopsis in allSynopses.items():
		for keyWord in keyWords:

			## Get gene lists + disorder information, if a certain condition is met.
			if isKeyWordInAnySymptom(keyWord.lower(), synopsis):

				associatedGenes = set(getGeneListFromSynopsis(synopsis, phenoMimsToGenesDict, officialGeneList))
				# brainGenes = []
				# associatedGenes, actualInheritance, _ = getSynopsisInfo(synopsis, brainGenes, phenoMimsToGenesDict)

				geneSets[keyWord] |= set(associatedGenes)

	# Write out each gene set.
	for keyWord in keyWords:
		genes = geneSets[keyWord]
		with open(outputFolder + keyWord + ".tsv", "w") as out:
			out.write("\n".join(sorted(genes)))

	return

# Return a dict of keyterms to genes.
def getGeneSetsFromFolder(path):

	geneSets = {} # defaultdict(set)

	filenames = os.listdir(path)

	for filename in filenames:

		if ".tsv" in filename:

			with open(path + filename, "r") as file:
				genes = {gene.strip() for gene in file}
				geneSets[filename] = genes

	return geneSets

def compareSets():

	GTRfilesPath = PROJECT + "Results/GeneSets/GTRgeneLists/"
	ourFilesPath = PROJECT + "Results/GeneSets/Ours/"

	GTRgeneSets = getGeneSetsFromFolder(GTRfilesPath)
	ourGeneSets = getGeneSetsFromFolder(ourFilesPath)

	assert(len(GTRgeneSets) == len(ourGeneSets))
	keyWords = GTRgeneSets.keys()

	with open("Results/FirstPassComparison.tsv", "w") as out:

		out.write("Set Name\tNum shared\tNumber just in GTR\tJust GTR\tNumber just in ours\tJust ours\n")
		for keyWord in sorted(keyWords):

			GTRgenes = GTRgeneSets[keyWord]
			ourGenes = ourGeneSets[keyWord]

			shared = ourGenes.intersection(GTRgenes)
			justGTR = GTRgenes.difference(ourGenes)
			justOurs = ourGenes.difference(GTRgenes)

			out.write("{}\t{}\t{}\t{}\t{}\t{}\n".format(keyWord, len(shared), len(justGTR), " | ".join(justGTR), len(justOurs), " | ".join(justOurs)))

###################################################################################################

### Next: filtering categories.

###################################################################################################

def getGeneSet():

	conceptsToTheirCode, codesToSynonyms = getAllSynonyms()

	allSynopses = getAllSynopses()
	officialGeneList = getOfficialGenes()

	print ("Parsing synopses...")
	phenoMimsToGenesDict = makeMIMdict()

	responses = set()
	response = None

	while response != "done":

		question = [inquirer.Text("Keyword", message = "Please enter your search term, or 'done'"
					),
		]
		answer = inquirer.prompt(question)
		response = answer["Keyword"]

		if response != "done":
			responses.add(response)
	print("using the keyWords:", responses)

	keyWords = set(responses)
	for response in responses:
		### If a term doesn't have a synonym, it won't be in conceptsToTheirCode
		if response.lower() in conceptsToTheirCode:
			code = conceptsToTheirCode[response.lower()]
			keyWords |= set(codesToSynonyms[code])
			print ("For {}, found the synonyms {}.".format(response, codesToSynonyms[code]))

	# foundGenes = set()
	foundGenesToKeyWords = defaultdict(set)
	categoriesToGenes = defaultdict(set)

	dontWant = ["molecularBasis", "miscellaneous", "inheritance"]

	### Get the info
	for _, synopsis in allSynopses.items():
		for keyWord in keyWords:

			## Get gene lists + disorder information, if a certain condition is met.
			if isKeyWordInAnySymptom(keyWord.lower(), synopsis):

				currentCategories = set()

				associatedGenes = set(getGeneListFromSynopsis(synopsis, phenoMimsToGenesDict, officialGeneList))
				# foundGenes |= associatedGenes
				for gene in associatedGenes:
					foundGenesToKeyWords[gene].add(keyWord)
				for aspect in synopsis:
					if isinstance(synopsis[aspect], str) and "{" in synopsis[aspect] and aspect not in dontWant:
						currentCategories.add(aspect)
				for gene in associatedGenes:
					for category in currentCategories:

						categoriesToGenes[category].add(gene)

	### Let the user remove categories:
	categoryToExclude = None
	excludedCategories = set()
	excludedGenes = set()
	while categoryToExclude !=  "No, these are great!":

		genesToExclude = categoriesToGenes[categoryToExclude]

		for geneToExclude in genesToExclude:

			excludedGenes.add(geneToExclude)
			excludedCategories.add(categoryToExclude)
			# foundGenes.remove(geneToExclude)
			del foundGenesToKeyWords[geneToExclude]
			if categoryToExclude:
				excludedCategories.add(categoryToExclude)
			for category, genes in categoriesToGenes.items():

				categoriesToGenes[category] = {gene for gene in genes if gene != geneToExclude}
		### If we removed the last gene from a certain category, we get rid of it:
		categoriesToGenes = {category: genes for category, genes in categoriesToGenes.items() if len(genes) > 0}

		question = [
	 	inquirer.List("Exclude", message = "There are currently {} genes. Would you like to exclude any categories?".format(len(foundGenesToKeyWords)),
	 				choices = ["No, these are great!"] + list(categoriesToGenes.keys()),
					),
		]
		answer = inquirer.prompt(question)
		categoryToExclude = answer["Exclude"]

	# print ("{} categories were excluded, removing {} total genes.".format(len(excludedCategories), len(excludedGenes)))
	# print ("The remaining categories are:")
	# print (" | ".join(sorted(categoriesToGenes.keys())))
	print ("Final gene list:")
	print (sorted(foundGenesToKeyWords))

	with open(PROJECT + "test.tsv", "w") as out:

		out.write("Gene\t{}\n".format("\t".join(sorted(keyWords))))
		for gene, keyWordsThatFlaggedIt in sorted(foundGenesToKeyWords.items()):

			out.write("{}\t{}\n".format(gene, "\t".join(["X" if keyWord in keyWordsThatFlaggedIt else "" for keyWord in sorted(keyWords)])))

if __name__ == '__main__':

	# OMIM-scraping. Do this every few months.
	# scrapeOmim()
	# createOneBigFile()
	# makeInheritanceFile()
	# makeInheritanceAndSystemsFile()
	# getGeneListsForEachCategory()
	# getSymptomsFromAllCategories()

	# getMultipleSpecifiedListsOfGenes()
	# getKeyTerms()

	# keyWords = getKeyTerms()

	# outputFolder = PROJECT + "Results/GeneSets/Ours/"
	# createOurGeneSets(keyWords, outputFolder)

	getGeneSet()
	# makeUniqueCodes()


# -*- coding: utf-8 -*-
"""
Andrew D. Rouillard
Computational Biologist
Target Sciences
GSK
andrew.d.rouillard@gsk.com
"""

import os
import numpy as np
import datasetIO
import featureselection

def main(validation_rep=0, validation_fold=0):

    # load target clusters
    print('loading target cluster assignments...', flush=True)
    target_cluster_path = 'targets/clusters/gene_cluster_byfamily.pickle'
    gene_cluster = datasetIO.load_clusterassignments(target_cluster_path)

    # load dataset info
    print('loading dataset info...', flush=True)
    dataset_info_path = 'datasets/nonredundant_features/dataset_info.txt'
    dataset_infos = datasetIO.load_datasetinfo(dataset_info_path)

    # load validation examples
    print('loading validation examples...', flush=True)
    validation_examples_path = 'targets/validation_examples/rep{0!s}_fold{1!s}.txt'.format(validation_rep, validation_fold)
    with open(validation_examples_path, mode='rt', encoding='utf-8', errors='surrogateescape') as fr:
        validation_examples = fr.read().split('\n')

    # specify results folder
    print('specifying results folder...', flush=True)
    results_folder = 'datasets/generalizable_features/rep{0!s}_fold{1!s}'.format(validation_rep, validation_fold)
    results_folder_parts = results_folder.split('/')
    for i in range(len(results_folder_parts)):
        results_folder_part = '/'.join(results_folder_parts[:i+1])
        if not os.path.exists(results_folder_part):
            os.mkdir(results_folder_part)

    # iterate over datasets
    print('iterating over datasets...', flush=True)
    for dataset_info in dataset_infos:

#        # just work with hpatissuesmrna for testing/debugging the pipeline
#        if dataset_info['abbreviation'] != 'hpatissuesmrna_cleaned':
#            print('skipping {0}. not in testing set...'.format(dataset_info['abbreviation']), flush=True)
#            continue

        # check if another python instance is already working on this dataset
        if os.path.exists('{0}/{1}_in_progress.txt'.format(results_folder, dataset_info['abbreviation'])):
            print('skipping {0}. already in progress...'.format(dataset_info['abbreviation']), flush=True)
            continue

        # log start of processing
        with open('{0}/{1}_in_progress.txt'.format(results_folder, dataset_info['abbreviation']), mode='wt', encoding='utf-8', errors='surrogateescape') as fw:
            print('working on {0}...'.format(dataset_info['abbreviation']), flush=True)
            fw.write('working on {0}...'.format(dataset_info['abbreviation']))

        # load dataset
        print('loading dataset...', flush=True)
        gene_atb = datasetIO.load_datamatrix(datasetpath=dataset_info['path'])

        # specify feature generalizability test parameters
        print('specifying feature generalizability test parameters...', flush=True)
        dataset_info['feature_generalizability_test_function'] = featureselection.univariate_grouppreserved_permtest
        dataset_info['feature_generalizability_test_permutations'] = 10000 # 100000
        dataset_info['feature_generalizability_test_targetclusterpath'] = target_cluster_path
        dataset_info['multiple_hypothesis_testing_correction_function'] = featureselection.multiple_hypothesis_testing_correction
        dataset_info['multiple_hypothesis_testing_correction_method'] = 'fdr_by'
        dataset_info['multiple_hypothesis_testing_correction_threshold'] = 0.05
        print('   feature_generalizability_test_function: {0}'.format(dataset_info['feature_generalizability_test_function']), flush=True)
        print('   feature_generalizability_test_permutations: {0!s}'.format(dataset_info['feature_generalizability_test_permutations']), flush=True)
        print('   feature_generalizability_test_targetclusterpath: {0}'.format(dataset_info['feature_generalizability_test_targetclusterpath']), flush=True)
        print('   multiple_hypothesis_testing_correction_function: {0}'.format(dataset_info['multiple_hypothesis_testing_correction_function']), flush=True)
        print('   multiple_hypothesis_testing_correction_method: {0}'.format(dataset_info['multiple_hypothesis_testing_correction_method']), flush=True)
        print('   multiple_hypothesis_testing_correction_threshold: {0!s}'.format(dataset_info['multiple_hypothesis_testing_correction_threshold']), flush=True)

        # exclude validation and unlabeled examples from significance calculation
        print('excluding validation and unlabeled examples from significance calculation...', flush=True)
        isvalidation = np.in1d(gene_atb.rowlabels, validation_examples)
        isunknown = gene_atb.rowmeta['class'] == 'unknown'
        istraintest = ~np.logical_or(isvalidation, isunknown)

        # compute feature generalizability with multiple hypothesis testing correction
        print('computing feature generalizability with multiple hypothesis testing correction...', flush=True)
        gene_atb.rowmeta['cluster'] = np.array([gene_cluster[g] if g in gene_cluster else -1 for g in gene_atb.rowlabels], dtype='int64')
        gene_atb.columnmeta['generalizability_test_statistic_values'], gene_atb.columnmeta['generalizability_pvalues'] = dataset_info['feature_generalizability_test_function'](X=gene_atb.matrix[istraintest,:], Y=(gene_atb.rowmeta['class'][istraintest] == 'positive'), G=gene_atb.rowmeta['cluster'][istraintest], numperm=dataset_info['feature_generalizability_test_permutations'])
        gene_atb.columnmeta['is_generalizable'], gene_atb.columnmeta['generalizability_pvalues_corrected'] = dataset_info['multiple_hypothesis_testing_correction_function'](gene_atb.columnmeta['generalizability_pvalues'], alpha=dataset_info['multiple_hypothesis_testing_correction_threshold'], method=dataset_info['multiple_hypothesis_testing_correction_method'])
        gene_atb.columnmeta['generalizability_correlation_sign'] = np.sign(gene_atb.columnmeta['generalizability_test_statistic_values'])
        if (gene_atb.columnmeta['generalizability_pvalues'] < 1/dataset_info['feature_generalizability_test_permutations']).any():
            print('    warning: not enough permutations to establish all pvalues...', flush=True)
        tobediscarded = np.logical_or(np.isnan(gene_atb.columnmeta['generalizability_pvalues']), np.isnan(gene_atb.columnmeta['generalizability_pvalues_corrected']))
        if tobediscarded.any():
            gene_atb.discard(tobediscarded, axis=1)

        # prioritize features
        print('prioritizing features...', flush=True)
        sortedindices = np.argsort(gene_atb.columnmeta['generalizability_pvalues_corrected'])
        gene_atb.reorder(sortedindices, axis=1)

        # save feature generalizability info
        print('saving feature generalizability info...', flush=True)
        with open('{0}/{1}_feature_generalizability_info.txt'.format(results_folder, dataset_info['abbreviation']), mode='wt', encoding='utf-8', errors='surrogateescape') as fw:
            writelist = ['dataset',
                         'abbreviation',
                         'feature',
                         'generalizability_test_statistic',
                         'generalizability_pvalue',
                         'generalizability_pvalue_corrected',
                         'is_generalizable',
                         'generalizability_correlation_sign',
                         'preferred_rowstat',
                         'similar_features']
            fw.write('\t'.join(writelist) + '\n')
            for j, feature in enumerate(gene_atb.columnlabels):
                writelist = [dataset_info['name'],
                             dataset_info['abbreviation'],
                             feature,
                             '{0:1.5g}'.format(gene_atb.columnmeta['generalizability_test_statistic_values'][j]),
                             '{0:1.5g}'.format(gene_atb.columnmeta['generalizability_pvalues'][j]),
                             '{0:1.5g}'.format(gene_atb.columnmeta['generalizability_pvalues_corrected'][j]),
                             '{0:1.5g}'.format(gene_atb.columnmeta['is_generalizable'][j]),
                             '{0:1.5g}'.format(gene_atb.columnmeta['generalizability_correlation_sign'][j]),
                             gene_atb.columnmeta['preferred_rowstat'][j],
                             gene_atb.columnmeta['similar_features'][j]]
                fw.write('\t'.join(writelist) + '\n')

        # discard features that are not generalizable
        print('discarding features that are not generalizable...', flush=True)
        tobediscarded = ~gene_atb.columnmeta['is_generalizable']
        if tobediscarded.any():
            # discard features
            print('    discarding {0!s} features. {1!s} features remaining...'.format(tobediscarded.sum(), (~tobediscarded).sum()), flush=True)
            gene_atb.discard(tobediscarded, axis=1)
        else:
            # keep all features
            print('    no features to discard. {0!s} features remaining...'.format(gene_atb.shape[1]), flush=True)

        # save if dataset has content
        print('saving if dataset has content...', flush=True)
        if gene_atb.shape[0] == 0 or gene_atb.shape[1] == 0:
            # no content
            print('    nothing to save...', flush=True)
        else:
            # save generalizable features
            print('    saving {0!s} generalizable features...'.format(gene_atb.shape[1]), flush=True)
            dataset_info['path'] = '{0}/{1}.txt.gz'.format(results_folder, dataset_info['abbreviation'])
            dataset_info['generalizable_genes'] = gene_atb.shape[0]
            dataset_info['generalizable_features'] = gene_atb.shape[1]
            dataset_info['feature_generalizability_test_function'] = 'featureselection.univariate_grouppreserved_permtest'
            dataset_info['multiple_hypothesis_testing_correction_function'] = 'featureselection.multiple_hypothesis_testing_correction'
            datasetIO.save_datamatrix(dataset_info['path'], gene_atb)
            datasetIO.append_datasetinfo('{0}/dataset_info.txt'.format(results_folder), dataset_info)

    print('done.', flush=True)

if __name__ == '__main__':
    main()


import pytest
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers

from drf_react_template.schema_form_encoder import (
    COLUMN_PROCESSOR_OVERRIDE_KEY,
    DEPENDENCY_CONDITIONAL_KEY,
    DEPENDENCY_DYNAMIC_KEY,
    DEPENDENCY_OVERRIDE_KEY,
    DEPENDENCY_SIMPLE_KEY,
    SCHEMA_OVERRIDE_KEY,
    UI_SCHEMA_OVERRIDE_KEY,
    ColumnProcessor,
    SchemaProcessor,
    UiSchemaProcessor,
)
from example.polls.serializers import (
    ChoiceSerializer,
    QuestionListSerializer,
    QuestionSerializer,
)

def test_question_and_choice_retrieve_schema(
    question_and_choice_retrieve_expected_schema,
):
    result = SchemaProcessor(QuestionSerializer(), {}).get_schema()
    assert result == question_and_choice_retrieve_expected_schema

def test_question_and_choice_retrieve_ui_schema(
    question_and_choice_retrieve_expected_ui_schema,
):
    result = UiSchemaProcessor(QuestionSerializer(), {}).get_ui_schema()
    assert result == question_and_choice_retrieve_expected_ui_schema

def test_question_and_choice_list(question_and_choice_list_expected_schema):
    result = ColumnProcessor(QuestionListSerializer(), {}).get_schema()
    assert result == question_and_choice_list_expected_schema

@pytest.mark.parametrize(
    ['class_name', 'form_name'],
    (
        ['TitleSerializer', 'Title'],
        ['Title', 'Title'],
        ['TitleAndSubtitle', 'Title And Subtitle'],
    ),
)
def test_serializer_title(class_name, form_name):
    ChoiceSerializer.__name__ = class_name

    result = SchemaProcessor(ChoiceSerializer(), {}).get_schema()
    assert result['title'] == form_name

def test_serializer_no_title():
    class NoTitleSerializer(ChoiceSerializer):
        def __init__(self, **kwargs):
            super().__init__(label='', **kwargs)

    result = SchemaProcessor(NoTitleSerializer(), {}).get_schema()
    assert result['title'] == ''

def test_field_label():
    new_label = 'test_label'

    class LabelSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(label=new_label)

    result = SchemaProcessor(LabelSerializer(), {}).get_schema()
    assert result['properties']['choice_text']['title'] == new_label

    result = ColumnProcessor(LabelSerializer(), {}).get_schema()
    assert result[0]['title'] == new_label

def test_field_hidden_label():
    new_label = 'test_label'
    style_dict = {'ui:options': {'label': False}}

    class LabelSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(label=new_label, style=style_dict)

    result = SchemaProcessor(LabelSerializer(), {}).get_schema()
    assert result['properties']['choice_text']['title'] == new_label

    result = UiSchemaProcessor(LabelSerializer(), {}).get_ui_schema()
    assert result['choice_text'] == style_dict

def test_read_only():
    class ReadOnlySerializer(ChoiceSerializer):
        choice_text = serializers.CharField(read_only=True)

    result = SchemaProcessor(ReadOnlySerializer(), {}).get_schema()
    assert 'choice_text' not in result['properties']

    result = UiSchemaProcessor(ReadOnlySerializer(), {}).get_ui_schema()
    assert 'choice_text' not in result

    result = ColumnProcessor(ReadOnlySerializer(), {}).get_schema()
    assert not any(d['key'] == 'choice_text' for d in result)

def test_required():
    class RequiredSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(required=False)

    result = SchemaProcessor(RequiredSerializer(), {}).get_schema()
    assert 'choice_text' not in result['required']

def test_allow_null():
    class AllowNullSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(allow_null=True)

    result = SchemaProcessor(AllowNullSerializer(), {}).get_schema()
    assert len(result['properties']['choice_text']['type']) == 2
    assert 'null' in result['properties']['choice_text']['type']

def test_default():
    new_default = 'default'

    class DefaultSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(default=new_default)

    result = SchemaProcessor(DefaultSerializer(), {}).get_schema()
    assert result['properties']['choice_text']['default'] == new_default
    assert 'choice_text' not in result['required']

def test_choice_custom_widget_and_type():
    new_widget = 'CustomWidget'
    new_type = 'CustomType'

    class CustomWidgetSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(
            style={'ui:widget': new_widget, 'schema:type': new_type}
        )

    result = SchemaProcessor(CustomWidgetSerializer(), {}).get_schema()
    assert result['properties']['choice_text']['type'] == new_type

    result = UiSchemaProcessor(CustomWidgetSerializer(), {}).get_ui_schema()
    assert result['choice_text']['ui:widget'] == new_widget

def test_choice_placeholder():
    new_placeholder = 'new_placeholder'

    class PlaceholderSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(style={'ui:placeholder': new_placeholder})

    result = UiSchemaProcessor(PlaceholderSerializer(), {}).get_ui_schema()
    assert result['choice_text']['ui:placeholder'] == new_placeholder

def test_choice_text_area_rows():
    widget = 'textarea'
    ui_options = {'rows': 8}

    class TextAreaWidgetSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(
            style={
                'ui:widget': widget,
                'ui:options': ui_options,
            }
        )

    result = UiSchemaProcessor(TextAreaWidgetSerializer(), {}).get_ui_schema()
    assert result['choice_text']['ui:widget'] == widget
    assert result['choice_text']['ui:options'] == ui_options

def test_choice_disabled():
    class DisabledSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(style={'ui:disabled': 'true'})

    result = UiSchemaProcessor(DisabledSerializer(), {}).get_ui_schema()
    assert result['choice_text']['ui:disabled'] == 'true'

def test_question_list_sort():
    order = 'ascend'

    class QuestionListSortSerializer(QuestionListSerializer):
        pub_date = serializers.DateField(style={'schema:sort': order})

    result = ColumnProcessor(QuestionListSortSerializer(), {}).get_schema()
    assert result[1]['defaultSortOrder'] == order

def test_question_list_sort_bad_key():
    order = 'BAD_KEY'

    class QuestionListSortSerializer(QuestionListSerializer):
        pub_date = serializers.DateField(style={'schema:sort': order})

    with pytest.raises(ValueError):
        ColumnProcessor(QuestionListSortSerializer(), {}).get_schema()

def test_choice_schema_override():
    title_override = 'Override'

    class SchemaOverrideSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(
            style={SCHEMA_OVERRIDE_KEY: {'type': 'string', 'title': title_override}}
        )

    result = SchemaProcessor(SchemaOverrideSerializer(), {}).get_schema()
    assert result['properties']['choice_text']['title'] == title_override

def test_choice_ui_schema_override():
    ui_override = {'ui:widget': 'updown'}

    class UiSchemaOverrideSerializer(ChoiceSerializer):
        choice_text = serializers.CharField(style={UI_SCHEMA_OVERRIDE_KEY: ui_override})

    result = UiSchemaProcessor(UiSchemaOverrideSerializer(), {}).get_ui_schema()
    assert result['choice_text'] == ui_override

def test_question_and_choice_list_override():
    title_override = 'Override'

    class ListOverrideSerializer(QuestionListSerializer):
        question_text = serializers.CharField(
            style={
                COLUMN_PROCESSOR_OVERRIDE_KEY: {
                    'title': title_override,
                    'dataIndex': 'question_text',
                    'key': 'question_text',
                }
            }
        )

    result = ColumnProcessor(ListOverrideSerializer(), {}).get_schema()
    assert result[0]['title'] == title_override

def test_serializer_level_style():
    ui_template_key = 'ui:template'
    new_template = 'new_template'

    class CustomStyleSerializer(ChoiceSerializer):
        def __init__(self, *args, **kwargs):
            kwargs['style'] = {'ui:template': new_template}
            super().__init__(*args, **kwargs)

    result = UiSchemaProcessor(CustomStyleSerializer(), {}).get_ui_schema()
    assert result[ui_template_key] == new_template

def test_choice_schema_dependency_key_error():
    class SchemaKeyErrorDependencySerializer(ChoiceSerializer):
        choice_text = serializers.CharField(
            style={DEPENDENCY_SIMPLE_KEY: 'votes', DEPENDENCY_OVERRIDE_KEY: {}}
        )

    with pytest.raises(KeyError):
        SchemaProcessor(SchemaKeyErrorDependencySerializer(), {}).get_schema()

@pytest.mark.parametrize(
    ['dependencies'],
    (['votes'], [['votes']]),
)
def test_choice_schema_simple_unidirectional_dependency(dependencies):
    class SchemaSimpleUnidirectionalDependencySerializer(ChoiceSerializer):
        choice_text = serializers.CharField(style={DEPENDENCY_SIMPLE_KEY: dependencies})

    result = SchemaProcessor(
        SchemaSimpleUnidirectionalDependencySerializer(), {}
    ).get_schema()
    assert 'votes' not in result['required']
    assert 'votes' in result['dependencies']['choice_text']
    ui_result = UiSchemaProcessor(
        SchemaSimpleUnidirectionalDependencySerializer(), {}
    ).get_ui_schema()
    assert DEPENDENCY_SIMPLE_KEY not in ui_result['votes']

def test_choice_schema_simple_bidirectional_dependency():
    class SchemaSimpleBidirectionalDependencySerializer(ChoiceSerializer):
        choice_text = serializers.CharField(style={DEPENDENCY_SIMPLE_KEY: 'votes'})
        votes = serializers.IntegerField(
            default=0, style={DEPENDENCY_SIMPLE_KEY: 'choice_text'}
        )

    result = SchemaProcessor(
        SchemaSimpleBidirectionalDependencySerializer(), {}
    ).get_schema()
    assert 'votes' not in result['required']
    assert 'choice_text' not in result['required']
    assert 'votes' in result['dependencies']['choice_text']
    assert 'choice_text' in result['dependencies']['votes']
    ui_result = UiSchemaProcessor(
        SchemaSimpleBidirectionalDependencySerializer(), {}
    ).get_ui_schema()
    assert DEPENDENCY_SIMPLE_KEY not in ui_result['votes']
    assert DEPENDENCY_SIMPLE_KEY not in ui_result['choice_text']

def test_choice_schema_conditional_dependency(choice_conditional_dependency_votes):
    class SchemaConditionalDependencySerializer(ChoiceSerializer):
        choice_text = serializers.CharField(
            style={DEPENDENCY_CONDITIONAL_KEY: ['votes']}
        )

    result = SchemaProcessor(SchemaConditionalDependencySerializer(), {}).get_schema()
    assert 'votes' not in result['required']
    assert 'votes' not in result['properties']
    assert result['dependencies'] == {
        'choice_text': choice_conditional_dependency_votes
    }
    ui_result = UiSchemaProcessor(
        SchemaConditionalDependencySerializer(), {}
    ).get_ui_schema()
    assert DEPENDENCY_CONDITIONAL_KEY not in ui_result['choice_text']

def test_choice_schema_dynamic_dependency(choice_dynamic_dependency_votes):
    class SchemaDynamicDependencySerializer(ChoiceSerializer):
        choice_text = serializers.ChoiceField(
            choices=(('yes', 'Yes'), ('no', 'No')),
            style={DEPENDENCY_DYNAMIC_KEY: {'yes': ['votes'], 'no': None}},
        )
        votes = serializers.IntegerField()

    result = SchemaProcessor(SchemaDynamicDependencySerializer(), {}).get_schema()
    assert result['dependencies'] == choice_dynamic_dependency_votes
    ui_result = UiSchemaProcessor(
        SchemaDynamicDependencySerializer(), {}
    ).get_ui_schema()
    assert DEPENDENCY_DYNAMIC_KEY not in ui_result['choice_text']

def test_choice_schema_dynamic_dependency_non_enum_field():
    class SchemaDynamicDependencySerializer(ChoiceSerializer):
        choice_text = serializers.CharField(
            style={DEPENDENCY_DYNAMIC_KEY: {'yes': ['votes'], 'no': None}},
        )

    with pytest.raises(KeyError):
        SchemaProcessor(SchemaDynamicDependencySerializer(), {}).get_schema()

def test_choice_schema_dynamic_dependency_bad_enum_key_in_style():
    class SchemaDynamicDependencySerializer(ChoiceSerializer):
        choice_text = serializers.ChoiceField(
            choices=(('yes', 'Yes'), ('no', 'No')),
            style={DEPENDENCY_DYNAMIC_KEY: {'yes': ['votes'], 'test': None}},
        )

    with pytest.raises(KeyError):
        SchemaProcessor(SchemaDynamicDependencySerializer(), {}).get_schema()

def test_choice_schema_override_dependency(choice_conditional_dependency_votes):
    class SchemaOverrideDependencySerializer(ChoiceSerializer):
        choice_text = serializers.CharField(
            style={DEPENDENCY_OVERRIDE_KEY: choice_conditional_dependency_votes}
        )

    result = SchemaProcessor(SchemaOverrideDependencySerializer(), {}).get_schema()
    # Since override doesn't do any processing this may result in a bad schema.
    assert 'votes' in result['properties']
    assert result['dependencies'] == {
        'choice_text': choice_conditional_dependency_votes
    }

def test_choice_schema_simple_dynamic_dependencies(choice_dynamic_dependency_votes):
    class SchemaSimpleDynamicDependencySerializer(ChoiceSerializer):
        choice_text = serializers.ChoiceField(
            choices=(('yes', 'Yes'), ('no', 'No')),
            style={DEPENDENCY_DYNAMIC_KEY: {'yes': ['votes'], 'no': None}},
        )
        votes = serializers.IntegerField()
        conditional = serializers.IntegerField(style={DEPENDENCY_SIMPLE_KEY: ['votes']})

    result = SchemaProcessor(SchemaSimpleDynamicDependencySerializer(), {}).get_schema()
    assert 'votes' in result['properties']
    assert 'votes' in result['dependencies']['conditional']
    assert (
        result['dependencies']['choice_text']
        == choice_dynamic_dependency_votes['choice_text']
    )

def test_extra_field_type(custom_field_type_expected_schema):
    class CustomFieldTypeSerializer(ChoiceSerializer):
        image_field = serializers.ImageField(required=True)
        uuid_field = serializers.UUIDField(required=True)

    result = SchemaProcessor(
        CustomFieldTypeSerializer(), {}, extra_types={'UUIDField': {'type': 'uuid'}}
    ).get_schema()
    assert result['properties'] == custom_field_type_expected_schema

def test_validation_schema():
    class MinSizeImageValidator:
        message = _('Image is too small, must be 1KB minimum.')
        code = 'image_min_1KB'

        def __call__(self, value):
            min_size = 1024  # 1KB
            if value.size < min_size:
                raise serializers.ValidationError(self.message, code=self.code)

    class CustomValidationSerializer(ChoiceSerializer):
        char_text = serializers.CharField(min_length=5, max_length=10)
        int_field = serializers.IntegerField(min_value=3, max_value=7)
        regex_field = serializers.RegexField(regex=r'^[a-zA-Z]+$', required=True)
        image_field = serializers.ImageField(
            required=True, validators=[MinSizeImageValidator]
        )
        list_field = serializers.ListField(
            child=serializers.IntegerField(min_value=0, max_value=100),
            min_length=1,
            max_length=5,
            allow_empty=True,
        )

    result = SchemaProcessor(CustomValidationSerializer(), {}).get_schema()

    assert result['properties']['regex_field']['pattern'] == '^[a-zA-Z]+$'
    assert result['properties']['char_text']['maxLength'] == 10
    assert result['properties']['char_text']['minLength'] == 5
    assert result['properties']['int_field']['maximum'] == 7
    assert result['properties']['int_field']['minimum'] == 3
    assert result['properties']['list_field']['maxLength'] == 5
    assert result['properties']['list_field']['minLength'] == 1
    assert result['properties']['list_field']['required'] is False

    ui_result = UiSchemaProcessor(CustomValidationSerializer(), {}).get_ui_schema()

    assert 'ui:custom-validators' not in ui_result['char_text']
    assert ui_result['image_field']['ui:custom-validators'] == [
        {'code': 'image_min_1KB', 'message': 'Image is too small, must be 1KB minimum.'}
    ]


import os, sys, pygame
from pygame.locals import *

pygame.init()
fpsClock = pygame.time.Clock()

gameScreen = pygame.display.set_mode((960, 960))
pygame.display.set_caption('Dragon Warrior Pygame')

overworld = []
with open('overworld.txt') as mapfile:
    overworld = list(map(int, mapfile.read().split('\n')))

tantagel = []
with open('tantagel.txt') as mapfile:
    tantagel = list(map(int, mapfile.read().split('\n')))

x_offset = 15
y_offset = 0
counter = 0

class Character:

	def __init__(self, x_position, y_position):
		self.x_position = x_position
		self.y_position = y_position
		self.sprite_frame = 0

def load_tileset(filename, width, height):
	#load image
	image = pygame.image.load(filename)
	#get size of image and store
	image_width, image_height = image.get_size()
	#initialize list to hold tiles
	tile_table = []
	#load tiles into list
	for tile_y in range(0, image_height//height): #split image into rows by tile height
		for tile_x in range(0, image_width//width):  #split rows into tiles by tile width
			rect = (tile_x*width, tile_y*height, width, height)  #grab a tile
			tile_table.append(image.subsurface(rect))  #add tile to the map list
	#return the list
	return tile_table

def load_map(mapname, x_off, y_off):
	# initialize variables
	x_off = x_off - 7  #x and y offsets to place character in the middle of the screen
	y_off = y_off - 7
	temp_map = mapname
	current_map = []
	map_width = mapname[-1]  #the last index in the list is the width of the map

	for a in range(0, 15):
		for b in range(0, 15):
			current_map.append(temp_map[x_off + (y_off * map_width)] - 1)
			x_off += 1
		y_off += 1
		x_off -= 15

	return current_map

def draw_map(current_map):
	x = 0
	x_position = 0
	y_position = 0

	for a in range(0, 15):
		for b in range(0, 15):
			gameScreen.blit(terrain_tiles[current_map[x]], (x_position, y_position))
			x_position += 64
			x += 1
		x_position = 0
		y_position += 64

def draw_characters():

	hero_frame_one = hero.sprite_frame
	hero_frame_two = hero.sprite_frame + 1

	if counter < 7:
		gameScreen.blit(char_sprites[hero_frame_one], (448, 448))
	elif counter >= 7:
		gameScreen.blit(char_sprites[hero_frame_two], (448, 448))

def do_movement(current_map):

	#create list of collision tiles
	uncrossable_tiles = []
	with open('forbidden.txt') as mapfile:
	    uncrossable_tiles = list(map(int, mapfile.read().split('\n')))

	for event in pygame.event.get():
		if event.type == KEYDOWN:
			if event.key == K_UP:
				hero.sprite_frame = 4
				if current_map[97] not in uncrossable_tiles:
					hero.y_position -= 1
			elif event.key == K_DOWN:
				hero.sprite_frame = 0
				if current_map[127] not in uncrossable_tiles:
					hero.y_position += 1
			elif event.key == K_LEFT:
				hero.sprite_frame = 2
				if current_map[111] not in uncrossable_tiles:
					hero.x_position -= 1
			elif event.key == K_RIGHT:
				hero.sprite_frame = 6
				if current_map[113] not in uncrossable_tiles:
					hero.x_position += 1
			elif event.key == K_ESCAPE:
				pygame.quit()
				sys.exit()

terrain_tiles = load_tileset("tiles.png", 64, 64)
char_sprites = load_tileset("tchars.png", 64, 64)

hero = Character(46, 47)

while True:

	level_map = load_map(overworld, hero.x_position, hero.y_position)

	draw_map(level_map)

	draw_characters()

	pygame.display.update()

	do_movement(level_map)

	if counter < 15:
		counter += 1
	elif counter >= 15:
		counter = 1

	fpsClock.tick(30)


from src.processor import Session
from dateutil.parser import parse

record = {'ip': '123.123.123.1', 'date': '0366-12-24', 'time': '23:59:59'}
baddate = {'ip': '123.123.123.1', 'date': '0x66-12-24', 'time': '23:59:59'}
badtime = {'ip': '123.123.123.1', 'date': '0366-12-24', 'time': 'z3:59:59'}

def test_session_init_bad_date():
    try:
        now = parse(baddate['date'] + ' ' + baddate['time'])
        sess = Session(start=now)
    except ValueError:
        assert True
    else:
        assert False

def test_session_init_bad_time():
    try:
        now = parse(badtime['date'] + ' ' + badtime['time'])
        sess = Session(start=now)
    except ValueError:
        assert True
    else:
        assert False

def test_session_init():
    now = parse(record['date'] + ' ' + record['time'])
    sess = Session(start=now, sleep=0)

    assert sess.start == now
    assert sess.end == now
    assert sess.count == 1

def test_date_output_format():
    now = parse(record['date'] + ' ' + record['time'])
    expected = record['date'] + ' ' + record['time']

    sess = Session(start=now, sleep=0)
    output = sess._format_dt_for_output(sess.start)

    assert output == expected


from flask import Flask,render_template
app=Flask(__name__)

@app.route('/')
@app.route('/user/<username>')
def index(username):
    user = {
        'D':{
            'name':'Mr D',
            'age':100,
            'gender':'male'
        },
        'A':{'name':'Mr.A',
             'age':200,
             'gender':'Gay'
             },
        'B':{'name':'Ms B',
             'age':12,
             'gender':'Lesbian',
             },
        'C':{'name':'Ms C',
             'age':13,
             'gender':'Male'}
    }

    if username in user.keys():
        post = user[username]
        return render_template('exhigh.html',post=post)
    else:
        return 'User not found'

if __name__ == '__main__':
    app.run(debug=True)


# Title: Clean_Extracted_Tables

# dir = '/Volumes/C/projects/SMART Plan Systemwide STOPS/AC_MDTEXIST#MDTNOB#MDTBLD_STOPSY2015Results'
# txt_dir = "txt_extract"
# csv_dir = "csv_dir"

import os, csv

###############################################################################
# Define functions
###############################################################################

# Function extract tables (*.txt) from report (*.prn)
def extractTables(report_file, tables, output_dir):
    # initialize
    count = 0
    found_table_no = 0
    is_outfile_open = 'False'
    table_end = '-------------------------------------------------------------------------------------------------------------------------------------'

    with open(report_file, "rt") as in_file:
        for line in in_file:

            if found_table_no < len(tables):
                table_number = 'Table' + str(tables[found_table_no]).rjust(9,)

            #  At Table start
            if (line.find(table_number, 0, 14) != -1):
                count = 1
                found_table_no += 1
                output_table = table_number + '.txt' #'Table' + str(tables[found_table_no]) +'.txt'
                is_outfile_open = 'True'
                out_file = open(os.path.join(output_dir, output_table),'w')

            # At Table end
            if line.find(table_end, 0, 134) != -1:
                count = 0
                if is_outfile_open == 'True':
                    out_file.close()
                is_outfile_open = 'False'

            # Extract Table contents
            if count > 0:
                #table_raw.append(line)
                out_file.write(line)

# Function: Get list of text files
def getFiles(dir):
   table_files = []
   for file in os.listdir(dir):
        if file.startswith("Table") & file.endswith(".txt"):
            table_files.append(file)
   return table_files

# Function to extract data
def extractTableContents(dir_file, col_widths):
    if col_widths != None:
        extractTableContentsFixed(dir_file, col_widths)
    else:
        extractTableContentsDelimited(dir_file)

# Function to extract fixed column table data
def extractTableContentsFixed(dir_file, col_widths):
    table_rows = []

    with open (dir_file, 'rt') as in_file:

        # get rows
        for line in in_file:

            # check last column data to identify table
            last_col = line[ col_widths[len(col_widths) - 2] + 1 : col_widths[len(col_widths) - 1] ]
            data_col = last_col.lstrip()

            # identifier to remove header separator
            if (len(data_col) > 0) and  ('=' not in data_col) :

                # get row elements
                row_elements = []
                for e in range(len(col_widths)):
                    if e == 0:
                        row_elements.append(line[0: col_widths[e] -1])
                    else:
                        row_elements.append(line[col_widths[e - 1] : col_widths[e] - 1])

                # append row elements to table
                table_rows.append(row_elements)
                # print(table_rows)
    return table_rows

# Function to extract space delimited table data
def extractTableContentsDelimited(dir_file):
    table_rows = []
    found_header = 0
    with open (dir_file, 'rt') as in_file:

        # get rows
        for line in in_file:

            # check header separator (start of table)
            header_separator = "======"
            row_elements = []

            # get table header (fieldnames)
            if line.find(header_separator, len(header_separator)) != -1 and found_header == 0:
                fieldNames = prev_line.split(" ")
                if len(fieldNames) > 1: # at least 3 columns
                    found_header = 1
                    for e in fieldNames:
                        e_value = e.strip()
                        if len(e_value) > 0 and e_value != ":" :
                            row_elements.append(e_value)
                    table_rows.append(row_elements)

            # get table contents
            if line.find(header_separator, len(header_separator)) == -1 and found_header == 1:
                elements = line.split(" ")
                if len(elements) > 1: # at least 3 columns
                    for e in elements:
                        e_value = e.strip()
                        if len(e_value) > 0 and e_value != ":":
                            if e_value == "-":
                                e_value = 0
                            row_elements.append(e_value)
                    table_rows.append(row_elements)

            # store previous line (to get header)
            prev_line = line

    return table_rows

# Funtion: Generate csv filename from raw table name
def get_csv_filename(file):
    file_no_ext = file.split(".txt")[0]
    csv_file_name = os.path.join(dir, file_no_ext + '.csv')
    return csv_file_name

# Function: write table to csv
def write_table_to_csv(data, csv_file_name):
    with open(csv_file_name, 'w') as csvfile:
        writer = csv.writer(csvfile)
        for row in data:
            writer.writerow(row)

# Function to create directory
def create_directory(dir, folder):
    directory = os.path.join(dir, folder)
    if not os.path.exists(directory):
        os.makedirs(directory)

# Function to get all files
def getAllFiles(dir):
   table_files = []
   for file in os.listdir(dir):
        table_files.append(file)
   return table_files


# coding: UTF-8
import requests
import json
from scripts.data_process import extractor

# website-background
def get_default_search_word() -> dict:
    Data = requests.get('https://api.bilibili.com/x/web-interface/search/default')
    JsonData = json.loads(Data.text)
    return {'response_code': Data.status_code, 'return_code': JsonData['code'],
            **extractor(data = JsonData['data'], dicts = {'name': 'show_name', 'url': 'url'})}

def get_background_image() -> dict:
    Data = requests.get('https://api.bilibili.com/x/web-show/res/locs?pf=0&ids=142')
    JsonData = json.loads(Data.text)
    return {'response_code': Data.status_code, 'return_code': JsonData['code'],
            'background': JsonData['data']['142'][0]['pic'], 'title': JsonData['data']['142'][0]['litpic']}

# video-background
def get_danmaku_preview(aid: int) -> dict:
    Data = requests.get(f'https://api.bilibili.com/x/v2/dm/ajax?aid={aid}')
    JsonData = json.loads(Data.text)
    return {'response_code': Data.status_code, 'return_code': JsonData['code'], 'danmaku_list': JsonData['data']}

# tag-api
def get_tag_id(name: str) -> dict:
    Data = requests.get(f'https://api.bilibili.com/x/tag/info?tag_name={name}')
    JsonData = json.loads(Data.text)
    return {'response_code': Data.status_code, 'return_code': JsonData['code'],
            'tag_id': JsonData['data']['tag_id']}

def get_tag_video(tag_id: int, page_size: int = 10) -> dict:
    BaseUrl = 'https://api.bilibili.com/x/web-interface/tag/top?pn={}&ps={}&tid={}'
    Page = 1
    Data = requests.get(BaseUrl.format(Page, page_size, tag_id))
    VideoInfoDict = {'aid': 'aid', 'bvid': 'bvid', 'title': 'title', 'tname': 'tname',
                     'copyright': 'copyright', 'upload_time': 'pubdate', 'introduction': 'desc'}
    VideoDataDict = {'view': 'view', 'danmaku': 'danmaku', 'like': 'like', 'dislike': 'dislike',
                     'reply': 'reply', 'coin': 'coin', 'collect': 'favorite', 'share': 'share'}

    while Data.text != '':
        Page += 1
        JsonData = json.loads(Data.text)
        VideoList = []

        for Video in JsonData['data']:
            VideoList.append({**extractor(data = Video, dicts = VideoInfoDict),
                              **extractor(data = Video['stat'], dicts = VideoDataDict)})
        Data = requests.get(BaseUrl.format(Page, page_size, tag_id))
        yield {'response_code': Data.status_code, 'return_code': JsonData['code'], 'videos': VideoList}


# -*- coding: utf-8 -*-

import os
import sys
from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand

version = "0.0.4"

def read_file(filename):
    if os.path.exists(filename):
        with open(filename) as fd:
            return fd.read()
    return ''

class PyTest(TestCommand):
    user_options = [
        ('pytest-args=', 'a', "Arguments to pass to py.test"),
    ]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        self.pytest_args = []

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        # import here, cause outside the eggs aren't loaded
        import pytest  # NOQA
        errno = pytest.main(self.pytest_args or ['--cov-report=term-missing'])
        sys.exit(errno)

setup(
    name='gtd',
    version=version,
    description="Manages a simple log book, to Get Things Done",
    long_description=read_file('README.rst'),
    cmdclass={'test': PyTest},
    classifiers=[
        # 'Development Status :: 1 - Planning',
        # 'Development Status :: 2 - Pre-Alpha',
        'Development Status :: 3 - Alpha',
        # 'Development Status :: 4 - Beta',
        # 'Development Status :: 5 - Production/Stable',
        # 'Development Status :: 6 - Mature',
        # 'Development Status :: 7 - Inactive',
        # 'Programming Language :: Python :: 2.7',
        # 'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: Implementation :: CPython',
        # 'Programming Language :: Python :: Implementation :: PyPy',
        'License :: OSI Approved :: MIT License',
        'Topic :: Software Development :: Documentation',
        'Operating System :: OS Independent',
    ],
    keywords='logbook scheduler todo backlog',
    author='Miguel Ángel García',
    author_email='miguelangel.garcia@gmail.com',
    url='https://github.com/magmax/logbook',
    license='MIT',
    packages=find_packages(exclude=['tests']),
    include_package_data=True,
    zip_safe=False,
    extras_require={
        'develop': [
            'pytest        >= 3.0.6',
            'pytest-cov    >= 2.4.0',

            'coveralls     >= 1.1',

            'flake8        >= 3.3.8',
        ],
    },
    install_requires=[
        'pyyaml'
    ],
    entry_points={
        'console_scripts': [
            'gtd = gtd.__main__:main',
        ],
    },
)


import glob
import os
import tensorflow as tf

def extract_weights(labels, params):
	""" Extracts the weights from the labels dictionary. """

	if (params['weighted_mmd'] == 'True') and (params['balanced_weights'] == 'True'):
		sample_weights_pos = tf.expand_dims(tf.identity(labels['balanced_weights'][:, 0]), axis=-1)
		sample_weights_neg = tf.expand_dims(tf.identity(labels['balanced_weights'][:, 1]), axis=-1)
		sample_weights = tf.expand_dims(tf.identity(labels['balanced_weights'][:, 2]), axis=-1)
	else:
		sample_weights_pos = tf.expand_dims(tf.identity(labels['unbalanced_weights'][:, 0]), axis=-1)
		sample_weights_neg = tf.expand_dims(tf.identity(labels['unbalanced_weights'][:, 1]), axis=-1)
		sample_weights = tf.expand_dims(tf.identity(labels['unbalanced_weights'][:, 2]), axis=-1)

	return sample_weights, sample_weights_pos, sample_weights_neg

def flatten_dict(dd, separator='_', prefix=''):
	""" Flattens the dictionary with eval metrics """
	return {
		prefix + separator + k if prefix else k: v
		for kk, vv in dd.items()
		for k, v in flatten_dict(vv, separator, kk).items()
	} if isinstance(dd,
		dict) else {prefix: dd}

def cleanup_directory(directory):
	"""Deletes all files within directory and its subdirectories.

	Args:
		directory: string, the directory to clean up
	Returns:
		None
	"""
	if os.path.exists(directory):
		files = glob.glob(f"{directory}/*", recursive=True)
		for f in files:
			if os.path.isdir(f):
				os.system(f"rm {f}/* ")
			else:
				os.system(f"rm {f}")


"""
-- 다익스트라
- 한 노드에서 다른 나머지 노드로 가는 최소비용을 구해야할 때 -- 시작노드(기준)가 있을때
- dp 와 우선순위큐

- dp 배열 max로 초기화하고 , 시작노드만 0으로 갱신
- 연결 그래프 [시작노드] - (도착노드 , 비용 )..
- 우선순위큐 - 비용이 적은것부터 뽑고, dp배열을 최소로 갱신한 후 갱신한 노드와 값을 우큐에 넣음

import heapq
h=[]
heapq.heappush(h , (val , node))
heapq.heappop(h)

"""

# 연결 그래프 : 시작 노드 - (도착노드 , 비용 ) , (도착노드 , 비용) ..
# 거리 기록 배열 : d[max로 초기화], 시작노드를 0으로 하기
# 우선순위큐 : 최소 비용 , 처음에 시작 노드와 비용 0 넣기

import heapq

def dijkstra(graph, start, cnt, end):
    node_list = [[] for _ in range(cnt + 1)]
    INF = int(1e9)
    d = [INF for _ in range(cnt + 1)]
    h = []
    for g in graph:
        n1, n2, val = g
        node_list[n1].append((n2, val))

    # 비용 , 노드 순 - 비용 작은순으로 정렬
    heapq.heappush(h, (0, start))
    # 시작 노드 0 으로 초기화
    d[start] = 0

    while h:
        val, n1 = heapq.heappop(h)
        for next in node_list[n1]:
            n2, nval = next
            if d[n1] + nval < d[n2]:
                d[n2] = d[n1] + nval
                heapq.heappush(h, (d[n2], n2))
    print(d)
    return d[end]

dijkstra([[1 ,2, 2],
[1 ,3 ,3],
[2 ,3 ,2],
[1 ,4 ,1],
[1 ,5 ,10],
[2 ,4 ,2],
[3 ,4 ,1],
[3 ,5, 1],
[4, 5, 3]]  , 1, 5, 5)

s =[]
heapq.heappush(s , (-0,"n1"))
heapq.heappush(s , (-1,"n1"))
heapq.heappush(s , (-2,"n1"))

while s :
    print(heapq.heappop(s))


import opals
from opals import Import, Bounds
import os, csv
# By convention, we abbreviate matplotlib as mpl
import matplotlib as mpl
# select the graphics kit to use
mpl.use("TkAgg")
# import does not work recursively,
# so we need to import the sub-module, too.
# By convention, we name it plt.
import matplotlib.pyplot as plt

# change the working to the OPALS demo directory
os.chdir(os.path.join(opals.__path__[0], r'..\demo'))

hatches = ["/", "\\", "|", "-", "+", "x", "o"]
colors = ["b", "g", "r", "c", "m", "y", "k"]

# loop 3 lists at the same time
# built in function 'zip' returns a list of tuples
# where the i-th tuple contains the i-th element of each of the sequences passed as arguments to 'zip'
# the shortest list determines the number of loops

for dataset, hatch, color in zip(["G111", "G112", "G113"], hatches, colors):
    dataFn = dataset + ".las"
    odmFn = dataset + ".odm"
    boundsFn = dataset + "_bounds.xyz"

    imp = Import.Import()
    imp.inFile = dataFn
    imp.run()

    bnds = Bounds.Bounds()
    bnds.inFile = odmFn

    # set the boundary type to result in a tight fit
    # use the enumerator 'alphaShape' of the enumeration 'BoundaryType'
    bnds.boundsType = opals.Types.BoundaryType.alphaShape
    bnds.outFile = boundsFn
    bnds.run()

    # csv reader is an iterable object, reading a line of a text on each iteration
    # and returning another object that 'iterates over' the line split into tokens
    x, y = [], []
    with open(boundsFn) as fin:
        reader = csv.reader(fin, delimiter=' ', skipinitialspace=True)
        for line in reader:
            x.append(float(line[0]))
            y.append(float(line[1]))
    # plot a polygon using the lists of x and y, no fill but edges and a hatch
    plt.fill(x, y, hatch=hatch, color=color, fill=False, linewidth=2, label=dataset)

# Label the axis
plt.title('Strip boundaries')
plt.xlabel('Easting [m]')
plt.ylabel('Northing [m]')

# Insert a legend. Use defaults i.e.
# all plot-handles of the axis,
# with labels as specified in plt.fill(..).
plt.legend()
# Ensure equal scales for both coordinates.
plt.axis('scaled')
# Actually draw the figure.
plt.show()


from .madobj import Elem, Expr, ExprList, Sequence, Line

from .parser import parse, pyname, parses

def fromast(ast, name="mad", root=None, special=None):
    if special is None:
        special = []
    if root is None:
        root = Elem(name=name, _orig={}, _rorig={})
    macros = {}
    current_seq = None
    for st in ast:
        if st is None:
            continue
        # print 'st',st
        madname, value = st
        name = pyname(madname)
        kind = value[0]
        if hasattr(root, "_orig"):
            root._orig[madname] = name
        if hasattr(root, "_rorig"):
            root._rorig[name] = madname
        if kind == "variable":
            # print 'value',value
            if name in special:
                val = evaluate(value[1], None)
            else:
                val = evaluate(value[1], root)
            # print 'val',val
            root[name] = val
        elif kind == "expression":
            value = mkexpr(value[1])
            root[name] = value
        elif kind == "element":
            proto = value[1][0]
            if proto is not None:
                proto = pyname(proto)
            if proto == "macro":
                out = []
                for st in ast:
                    out.append(st)
                    if st == ("statement", ("value", "};")):
                        break
                macros[name] = out
            elif name == "endsequence":
                current_seq = None
            elif name == "return":
                break
            elif proto == "sequence":
                ne = Sequence(madname, parent=root["sequence"])
                current_seq = ne
            elif proto == "line":
                # right hand side of line statement is not interpret as assignment
                # later by fromast(attrs,....)
                ne = Line(name=madname, parent=root["line"], value=value[1][1][1])
            elif proto == None:
                ne = root[name]
            else:
                proelem = root[proto]
                ne = Elem(name=madname, parent=proelem)
            root[name] = ne
            attrs = value[1][1:]
            fromast(attrs, name=madname, root=ne, special=["refer", "From", "apertype"])
            if current_seq is not None and current_seq is not ne:
                current_seq.append(name, ne)
    # print macros
    return root

def load(fh, name=None, root=None):
    return fromast(parse(fh), name=name, root=root)

def loads(s, root=None):
    return fromast(parses(s), root=root)

_pyopen = open

def open(fn, root=None):
    if fn.endswith(".gz"):
        fh = gzip.open(fn)
    else:
        fh = _pyopen(fn)
    if root is not None:
        fn = "%s,%s" % (root.name, fn)
    return load(fh, name=fn, root=root)

def evaluate(value, lcl):
    if type(value) is str:
        try:
            value = pyname(value)
            if lcl:
                value = eval(value, Elem.gbl, lcl)
        except NameError as e:
            print("Warning", value, "not evaluated")
            print(e)
    elif type(value) is list:
        value = [evaluate(i, lcl) for i in value]
    return value

def mkexpr(value):
    if type(value) is str:
        value = pyname(value)
        value = Expr(value)
    elif type(value) is list:
        value = [pyname(v) for v in value]
        value = ExprList(*value)
    return value


from bibtex_print import bibtex_print

BIB_FILE_PATH = "test_data/pubs.bib"

def setup_function(function):
    function.bibtex_entries = bibtex_print.load_bibtex(BIB_FILE_PATH)

def test_print_title():
    good_titles = [
        "Deformably Registering and Annotating Whole CLARITY Brains to an Atlas via Masked LDDMM",
        'Global Brain Workshop 2016',
        'VESICLE : Volumetric Evaluation of Synaptic Interfaces using Computer vision at Large Scale',
        'Learning a Data-Driven Nosology:Progress, Challenges & Opportunities',
    ]

    for i, entry in enumerate(test_print_title.bibtex_entries):
        output = bibtex_print.print_title(entry)

        print(output)

        assert output != ""
        assert output in good_titles

def test_print_address():
    good_address = [
        "Kavli Neuroscience Discovery Institute & Center for Imaging Science",
        "Kavli Neuroscience Discovery Institute & Center for Imaging Science @ JHU",
    ]

    for i, entry in enumerate(test_print_address.bibtex_entries):
        output = bibtex_print.print_address(entry)

        assert output != ""

        if output is not None:
            assert output in good_address


def f(n):
    if n <= 18:
        return n + 3
    if n > 18 and (n % 3 == 0):
        return (n // 3) * f(n // 3) + n - 12
    if n > 18 and (n % 3 != 0):
        return f(n - 1) + n ** 2 + 5

count = 0
for i in range(1, 1000 + 1):
    if all(int(j) % 2 == 0 for j in str(f(i))):
        count += 1
print(count)


import logging
import webapp2
import re
from bs4 import BeautifulSoup
from google.appengine.api import urlfetch
from model import Deal

FEED_URL = 'https://www.ozbargain.com.au/deals/feed'

class Fetch(webapp2.RequestHandler):

    def get(self):
        self.response.headers['Content-Type'] = 'text/plain'

        try:
            result = urlfetch.fetch(FEED_URL, validate_certificate=True)
            if result.status_code == 200:
                soup = BeautifulSoup(result.content, 'xml')
                items = soup.findAll('item')
                for item in items:
                    link = item.link.text.encode('utf-8')
                    title = item.title.text.encode('utf-8')
                    description = item.description.text.encode('utf-8')

                    deal = Deal.get_by_id(link)
                    if deal == None:
                        deal = Deal(id=link, link=link, title=title,
                                    description=description, new=True)
                        keywords = re.sub(r'\W+', ' ', title)
                        keywords = keywords.lower().split(' ')
                        # To remove empty strings.
                        keywords = filter(None, keywords)
                        keywords = list(set(keywords))  # To remove duplicates.
                        deal.keywords = keywords
                        deal.put()

                self.response.write('OK')
            else:
                self.response.write('ERROR')
        except urlfetch.Error:
            logging.exception('Caught exception fetching data')
            self.response.write('EXCEPTION')

app = webapp2.WSGIApplication([
    ('/fetch', Fetch),
], debug=True)


def maximum_depth(root):
    """
    :type root: TreeNode
    :rtype: int
    """
    if root is None:
        return 0

    left_result = maximum_depth(root.left)
    right_result = maximum_depth(root.right)

    return max(left_result, right_result) + 1

def maximum_depth2(root, depth):
    """
    :type root: TreeNode
    :rtype: int
    """
    if root is None:
        return depth

    left_result = maximum_depth2(root.left, depth + 1)
    right_result = maximum_depth2(root.right, depth + 1)

    return max(left_result, right_result)