import pygame
import time
import math
from utils import scale_img, blit_rotate_center, blit_text_center
pygame.font.init()

# variable of the track
GRASS = scale_img(pygame.image.load('img/grass.jpg'), 2.5)
TRACK = scale_img(pygame.image.load('img/track.png'), 0.9)
HEDGE = scale_img(pygame.image.load('img/track-border.png'), 0.9)
HEDGE_MASK = pygame.mask.from_surface(HEDGE)
FINISH = pygame.image.load('img/finish.png')
FINISH_MASK = pygame.mask.from_surface(FINISH)
FINISH_POS = (130, 250)

# variable of the car
BARR = scale_img(pygame.image.load('pilots/barr.png'), 0.5)
FITT = scale_img(pygame.image.load('pilots/fitt.png'), 0.5)
MASS = scale_img(pygame.image.load('pilots/mass.png'), 0.5)
PIQ = scale_img(pygame.image.load('pilots/piq.png'), 0.5)
SEN = scale_img(pygame.image.load('pilots/sen.png'), 0.5)

# variable of the window
WIDTH, HEIGHT = TRACK.get_width(), TRACK.get_height()
WIN = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption('Brasil Racing')
MAIN_FONT = pygame.font.SysFont("comicsans", 44)
FPS = 60
# out from 2: (110, 70), old 6: (404, 680), old 12: (734, 399),
PATH = [(175, 119), (56, 133), (70, 481), (318, 731), (418, 521), (507, 475), (600, 551), (613, 715), (736, 713), (611, 357), (409, 343), (433, 257), (697, 258), (738, 123), (581, 71), (303, 78), (275, 377), (176, 388), (178, 260)]
# laps = 2

# level class
class GameInfo:
    LEVELS = 10

    def __init__(self, level=1):
        self.level = level
        self.started = False
        self.level_start_time = 0

    def next_level(self):
        self.level += 1
        self.started = False

    def reset(self):
        self.level = 1
        self.started = False
        self.level_start_time = 0

    def game_finished(self):
        return self.level > self.LEVELS

    def start_level(self):
        self.started = True
        self.level_start_time = time.time()

    def get_level_time(self):
        if not self.started:
            return 0
        return round(time.time() - self.level_start_time)

# draw the pilots
class AbstractCar: # not to be initiate directly
    def __init__(self, max_vel, rotation_vel):
        self.img = self.IMG
        self.max_vel = max_vel
        self.rotation_vel = rotation_vel
        self.vel = 0
        self.angle = 0
        self.x, self.y = self.START_POS
        self.acc = 0.5

    def rotate(self, left=False, right=False):
        if left:
            self.angle += self.rotation_vel
        elif right:
            self.angle -= self.rotation_vel

    def draw(self, win):
        blit_rotate_center(win, self.img, (self.x, self.y), self.angle)

    def move_forward(self):
        self.vel = min(self.vel + self.acc, self.max_vel)
        self.move()

    def move_backward(self):
        self.vel = max(self.vel - self.acc, -self.max_vel/2)
        self.move()

    def move(self):
        radians = math.radians(self.angle)
        vertical = math.cos(radians) * self.vel
        horizontal = math.sin(radians) * self.vel

        self.y -= vertical
        self.x -= horizontal

    def collide(self, mask, x=0, y=0):
        car_mask = pygame.mask.from_surface(self.img)
        offset = (int(self.x - x), int(self.y - y))
        poi = mask.overlap(car_mask, offset)
        return poi

    def reset(self):
        self.x, self.y = self.START_POS
        self.angle = 0
        self.vel = 0

# player class
class Player(AbstractCar):
    IMG = SEN
    START_POS = (180, 200)

    def reduce_speed(self):
        # self.acc = 0
        self.vel = max(self.vel - self.acc/2, 0)
        self.move()

    def bounce(self):
        self.vel = -self.vel/2
        self.move()

# computer class
class Computer(AbstractCar):
    IMG = PIQ
    START_POS = (150, 200)

    def __init__(self, max_vel, rotation_vel, path=[]):
        super().__init__(max_vel, rotation_vel)
        self.path = path
        self.current_point = 0
        self.vel = max_vel

    def draw_points(self, win):
        for point in self.path:
            pygame.draw.circle(win, (255, 0, 0), point, 5)

    def draw(self, win):
        super().draw(win)
        # self.draw_points(win)

    def calculate_angle(self):
        target_x, target_y = self.path[self.current_point]
        x_diff = target_x - self.x
        y_diff = target_y - self.y

        if y_diff == 0:
            desired_radian_angle = math.pi/2
        else:
            desired_radian_angle = math.atan(x_diff/y_diff)

        if target_y > self.y:
            desired_radian_angle += math.pi

        difference_in_angle = self.angle - math.degrees(desired_radian_angle)
        if difference_in_angle >= 180:
            difference_in_angle -= 360

        if difference_in_angle > 0:
            self.angle -= min(self.rotation_vel, abs(difference_in_angle))
        else:
            self.angle += min(self.rotation_vel, abs(difference_in_angle))

    def update_path_point(self):
        target = self.path[self.current_point]
        rect = pygame.Rect(self.x, self.y, self.img.get_width(), self.img.get_height())

        if rect.collidepoint(*target):
           self.current_point += 1
            # if self.current_point == len(self.path):
              #  self.current_point = 0
    def move(self):
        if self.current_point >= len(self.path):
            return

        self.calculate_angle()
        self.update_path_point()
        super().move()

    def next_level(self, level):
        self.reset()
        self.vel = self.max_vel + (level - 1) * 0.2
        self.current_point = 0

# draw the objects
def draw(win, images, player, computer, game_info):
    for img, pos in images:
        win.blit(img, pos)

    level_text = MAIN_FONT.render(
        f"Level {game_info.level}", 1, (255, 255, 255))
    win.blit(level_text, (10, HEIGHT - level_text.get_height() - 70))

    time_text = MAIN_FONT.render(
        f"Time: {game_info.get_level_time()}s", 1, (255, 255, 255))
    win.blit(time_text, (10, HEIGHT - time_text.get_height() - 40))

    vel_text = MAIN_FONT.render(
        f"Vel: {round(player.vel, 1)}px/s", 1, (255, 255, 255))
    win.blit(vel_text, (10, HEIGHT - vel_text.get_height() - 10))

    player.draw(win)
    computer.draw(win)
    pygame.display.update()

# move player
def move_player(player):
    keys = pygame.key.get_pressed()
    moved = False

    if keys[pygame.K_a]:
        player.rotate(left=True)
    if keys[pygame.K_d]:
        player.rotate(right=True)
    if keys[pygame.K_w]:
        moved = True
        player.move_forward()
    if keys[pygame.K_s]:
        moved = True
        player.move_backward()

    if not moved:
        player.reduce_speed()

# collision detection
def handle_collision(player_car, computer_car, game_info):
    if player_car.collide(HEDGE_MASK) != None:
        player_car.bounce()

    computer_finish_poi_collide = computer_car.collide(
        FINISH_MASK, *FINISH_POS)
    if computer_finish_poi_collide != None:
        blit_text_center(WIN, MAIN_FONT, "You lost!")
        pygame.display.update()
        pygame.time.wait(5000)
        game_info.reset()
        player_car.reset()
        computer_car.reset()

    player_finish_poi_collide = player_car.collide(
        FINISH_MASK, *FINISH_POS)
    if player_finish_poi_collide != None:
        if player_finish_poi_collide[1] == 0:
            player_car.bounce()
        else:
            game_info.next_level()
            player_car.reset()
            computer_car.next_level(game_info.level)

# define race loop
race = True
clock = pygame.time.Clock()
images = [(GRASS, (0,0)), (TRACK, (0,0)), (HEDGE, (0,0)), (FINISH, FINISH_POS), (HEDGE, (0,0))]
player = Player(5, 5)
computer = Computer(5, 5, PATH)
game_info = GameInfo()

while race:
    clock.tick(FPS) # the loop can't run faster than 60 frame per second

    # blit to show the objects on screen / putting on order of later appear
    draw(WIN, images, player, computer, game_info)
    while not game_info.started:
        blit_text_center(
            WIN, MAIN_FONT, f"Press any key to start level {game_info.level}!")
        pygame.display.update()
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                break
            if event.type == pygame.KEYDOWN:
                game_info.start_level()

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            run = False
            break

    move_player(player)
    computer.move()
    handle_collision(player, computer, game_info)

    if game_info.game_finished():
        blit_text_center(WIN, MAIN_FONT, "You won the game!")
        pygame.time.wait(5000)
        game_info.reset()
        player.reset()
        computer.reset()
pygame.quit()


#!/usr/bin/python3

import asyncio
import os
from typing import Dict, List, Optional, Set, Tuple, Any, cast
from datetime import datetime, timedelta

import aiohttp
import requests

class PRStats(object):

    def __init__(
        self,
        username: str,
        access_token: str,
        session: aiohttp.ClientSession,
        repo_owner: str = None,
        startDateTime: datetime = datetime.now() - timedelta(days = 365),
        endDateTime: datetime = datetime.now(),
    ):
        self.username = username
        self.access_token = access_token
        self.session = session

        self.repo_owner = repo_owner
        self.startDateTime = startDateTime
        self.endDateTime = endDateTime
        self.maxNumberOfRepos = 100

        self._prContributions: Optional[int] = None

    async def to_str(self) -> str:
        """
        :return: summary of all available statistics
        """
        return f"""Pull Request Contributions: {await self.prContributions}"""

    @property
    async def prContributions(self) -> int:
        """
        :return: total number of pull request contributions the user has performed (approvals, etc.)
        """
        if self._prContributions is not None:
            return self._prContributions
        await self.get_stats()
        assert self._prContributions is not None
        return self._prContributions

    async def get_stats(self) -> None:
        """
        Get lots of summary statistics using one big query. Sets many attributes
        """
        self._prContributions = 0

        url = 'https://api.github.com/graphql'
        query = (
        f"""
        {{
          user(login: "{self.username}") {{
            contributionsCollection(from: "{self.startDateTime.isoformat(sep='T', timespec='seconds')}", to: "{self.endDateTime.isoformat(sep='T', timespec='seconds')}") {{
              pullRequestReviewContributionsByRepository(maxRepositories: {self.maxNumberOfRepos}) {{
                contributions {{
                  totalCount
                }}
                repository {{
                  nameWithOwner
                }}
              }}
            }}
          }}
        }}
        """
        )
        json = { 'query' : query}
        api_token = f"{self.access_token}"
        headers = {'Authorization': 'token %s' % api_token}
        r = requests.post(url=url, json=json, headers=headers)

        #print(f"query: {query}")

        #print(f"json_data {r.json()}")
        json_data = r.json()['data']['user']['contributionsCollection']['pullRequestReviewContributionsByRepository']
        if self.repo_owner is None:
            self._prContributions = sum([i['contributions']['totalCount'] for i in json_data])
        else:
            self._prContributions = sum([i['contributions']['totalCount'] for i in json_data if i['repository']['nameWithOwner'].startswith(self.repo_owner)])
        print(f'Number of reviews: {self._prContributions}')
        self._prContributions = self._prContributions

###############################################################################
# Main Function
###############################################################################

async def main() -> None:
    """
    Used mostly for testing; this module is not usually run standalone
    """
    access_token = os.getenv("ACCESS_TOKEN")
    user = os.getenv("GITHUB_ACTOR")
    if access_token is None or user is None:
        raise RuntimeError(
            "ACCESS_TOKEN and GITHUB_ACTOR environment variables cannot be None!"
        )
    async with aiohttp.ClientSession() as session:
        s = PRStats(user, access_token, session)
        print(await s.to_str())

if __name__ == "__main__":
    asyncio.run(main())


import dataclasses
from inspect import isfunction
from math import isnan
from typing import Any
from typing import Union

from fvalues import F

JSONValue = Union[
    str, int, float, bool, None, list["JSONValue"], dict[str, "JSONValue"]
]

def to_json_value(x: Any) -> JSONValue:
    if isinstance(x, dict):
        return {
            k if isinstance(k, str) else repr(k): to_json_value(v) for k, v in x.items()
        }
    if isinstance(x, (list, tuple, set)):
        return [to_json_value(v) for v in x]
    if isinstance(x, F):
        return {"__fstring__": to_json_value(x.parts)}
    if hasattr(x, "dict") and callable(x.dict):
        try:
            x = x.dict()
        except TypeError:  # raised if wrong number of arguments
            pass
        else:
            return to_json_value(x)
    if dataclasses.is_dataclass(x):
        return to_json_value(dataclasses.asdict(x))
    if isfunction(x):
        return dict(class_name=x.__class__.__name__, name=x.__name__)
    if isinstance(x, float):
        return None if isnan(x) else x
    if isinstance(x, (int, str, bool, type(None))):
        return x
    return repr(x)


import numpy as np
import cv2

cap = cv2.VideoCapture(0)

while(True):
    # Capture the video frame by frame
    ret, frame = cap.read()

    # convert to gray picture
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Display the picture
    cv2.imshow('frame',gray)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# release the capture when quit
cap.release()
cv2.destroyAllWindows()


#Given the results of oligo enrichments, as calculated by DESeq2, define windows
#of enriched oligos that encompass active RNA elements.  Then, once you have
#windows, define minimal sequences within those windows that represent the
#minimal active element. These would be sequences that are in common to every
#oligo within the window.

#Windows are defined by looking for an enriched oligo and extending a window if either
#(A) the next oligo is sig or
#(B) there are x sig oligos in a row after a gap of x oligos

#After defining these primary windows, two windows are merged together if
#the distance between them is less than  (width of the narrow window + 2)
#After merging, any window less than 10 oligos long is removed

import os
import pandas as pd
import sys
import gffutils
from itertools import groupby
from itertools import product
from operator import itemgetter
import gzip
from Bio import SeqIO
import pysam

def simpleroligos2genome(gff):
    #Just make a table that relates oligo position in the UTR (i.e. 1, 2, 3, 4, etc.)
    #to the start genomecoord of the oligo

    oligocoords = {} #{utr : {oligoposition : genomecoordofstart}}
    oligostrands = {} #{utr : strand}
    oligochrms = {} #{utr: chrom}

    print('Indexing gff...')
    gff_fn = gff
    db_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge', verbose = True)

    db = gffutils.FeatureDB(db_fn)
    print('Done indexing!')

    utrs = db.features_of_type('UTR')

    for utr in utrs:
        utrid = str(utr.attributes['gene_name'][0])
        if utrid not in oligocoords:
            oligocoords[utrid] = {}
        if utrid not in oligostrands:
            oligostrands[utrid] = utr.strand
        if utrid not in oligochrms:
            oligochrms[utrid] = utr.chrom
        for oligo in db.children(utr, featuretype = 'oligo'):
            oligoid = int(str(oligo.id).split('.')[1])
            if oligo.strand == '+':
                genomecoord = int(oligo.start)
            elif oligo.strand == '-':
                genomecoord = int(oligo.stop)
            oligocoords[utrid][oligoid] = genomecoord
            if utrid == 'Afap1l1' and oligoid == 1:
                print('yo')

    #turn into df
    df = pd.concat({k: pd.DataFrame.from_dict(v, 'index') for k, v in oligocoords.items()}, axis = 0)
    #get indexes out as column names
    df.reset_index(level = 1, inplace = True)
    df.reset_index(level = 0, inplace = True)
    df.columns = ['genename', 'oligopos', 'genomecoord']
    df['strand'] = [oligostrands[x] for x in df['genename']]
    df['chrm'] = [oligochrms[x] for x in df['genename']]
    print(df.head())

    return df

def oligos2genome(gff):
	#We need some way to relate oligo positions, namely through their ID's (e.g. ENSMUSG00000025352.256|Gdf11)
	#to positions in the genome that they cover. One way to do this would be to, for each oligo,
	#make a list of coordinates that are tiled by the oligo: {utr : [list of exonic coordinates covered by utr's oligos]}
	#then, from that, make dictionary of {utr : {utr nt (as defined by getntcoverage (this goes from 1 to len(utr))) : genome coord}}

	#Gffs are annotations of oligo positions and were made by OligoPools_shortstep_260nt.py

	utrcoords = {} #{utr : [list of exonic coordinates covered by utr's oligos]}

	print('Indexing gff...')
	gff_fn = gff
	db_fn = os.path.abspath(gff_fn) + '.db'
	if os.path.isfile(db_fn) == False:
		gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge', verbose = True)

	db = gffutils.FeatureDB(db_fn)
	print('Done indexing!')

	utrs = db.features_of_type('UTR')

	for utr in utrs:
		utrid = str(utr.attributes['gene_name'][0])
		utrexoniccoords = []
		for oligo in db.children(utr, featuretype = 'oligo', level = 1):
			#Does this oligo cross a splice junction (i.e. does it have junctionpiece children?)
			if len(list(db.children(oligo, featuretype = 'junctionpiece'))) > 0:
				#yes there is a junctionpiece
				for jp in db.children(oligo, featuretype = 'junctionpiece'):
					jpnt = list(range(jp.start, jp.end + 1))
					utrexoniccoords += jpnt
			elif len(list(db.children(oligo, featuretype = 'junctionpiece'))) == 0:
				#no there is not a junctionpiece
				oligont = list(range(oligo.start, oligo.end + 1))
				utrexoniccoords += oligont

		#there's gonna be a lot of redundancy in utrexoniccoords because the oligos overlap
		utrexoniccoords = list(set(utrexoniccoords))
		if utr.strand == '+':
			utrexoniccoords = list(sorted(utrexoniccoords))
		elif utr.strand == '-':
			utrexoniccoords = list(reversed(sorted(utrexoniccoords)))

		utrcoords[utrid] = utrexoniccoords

	#OK now we need to go from utrcoords to utr_oligocoords {utr : {utr nt (as defined by getntcoverage (this goes from 1 to len(utr))) : genome coord}}

	utr_oligocoords = {} #{utr : {utr nt (as defined by getntcoverage (this goes from 1 to len(utr))) : genome coord}}
	for utr in utrcoords:
		genomecoords = utrcoords[utr]
		oligocoords = list(range(len(genomecoords) + 1))
		oligocoords2genomecoords = dict(zip(oligocoords, genomecoords))
		utr_oligocoords[utr] = oligocoords2genomecoords

	#turn utr_oligocoords into a df and return the df
	df = pd.concat({k: pd.DataFrame.from_dict(v, 'index') for k, v in utr_oligocoords.items()}, axis = 0)
	#get indexes out as column names
	df.reset_index(level = 1, inplace = True)
	df.reset_index(level = 0, inplace = True)
	df.columns = ['genename', 'utrcoord', 'genomecoord']

	return df

def definewindows(deseqtable, windowlengthminimum):
    enrichedwindows = {} #{gene : [(windowstart, windowstop)]}
    #The input for this function is a modified DESeq2 output table
    #It contains the following columns:
    #ensid, genename, baseMean, log2FoldChange, lfcSE, stat, pvalue, padj, sig, neuritelog2FC, expected
    #sig is either 'soma', 'neurite', or 'no'

    #Read file
    df = pd.read_csv(deseqtable, sep = '\t', header = 0, index_col = None)
    #ensid is of the form ENSMUSG00000.0
    #the number after the dot is the position of the oligo along the UTR
    df[['ensid', 'oligopos']] = df['ensid'].str.split('.', n = 1, expand = True)
    #Reorder columns
    df = df[['ensid', 'oligopos', 'genename', 'baseMean', 'log2FoldChange', 'lfcSE', 'stat', 'pvalue', 'padj', 'sig', 'neuritelog2FC', 'expected']]
    #Get rid of columns we don't need
    df = df[['genename', 'oligopos', 'sig']]
    #Convert oligopos to integer
    df = df.astype({'oligopos' : int})
    #Add in the start nucleotide position (0-based) of each oligo relative to the beginning of the first oligo for this gene
    #Oligos are 260 nt long, 4 nt step size
    oligopos = df.oligopos.tolist()
    utrcoord = [(x - 1) * 4 for x in oligopos]
    df = df.assign(utrcoord = utrcoord)
    #Arrange by genename then oligopos
    df = df.sort_values(by = ['genename', 'oligopos'], axis = 0)

    #OK, now for each gene, define windows of significance
    #Walk along UTR. oligopos = x. If oligo at x sig == 'neurite' oligo, open window
    #If x + 1 oligo is sig, extend window and repeat.
    #If not, go until you find a sig. Distance between current pos and next sig is y
    #If -all- of the oligos between x + y and x + 2y are sig, then extend window and go back to the beginning.
    #If not, window ends.
    #If window length passes threshold, keep it.

    genes = df.genename.tolist()
    genes = list(set(genes))
    genedfs = [] #list of individual genedfs
    for gene in genes:
        genedf = df.loc[df['genename'] == gene]
        alloligopos = genedf.oligopos.tolist()
        sigoligopos = genedf.loc[genedf['sig'] == 'neurite'].oligopos.tolist()

        inwindow = False
        oligoposinwindow = [] #oligo positions in -any- window
        currentidx = 0
        for idx, oligopos in enumerate(alloligopos):
            #print(gene, idx, oligopos, inwindow, oligoposinwindow)
            if idx < currentidx:
                continue
            if inwindow == False:
                if oligopos not in sigoligopos:
                    continue
                elif oligopos in sigoligopos:
                    inwindow = True
                    oligoposinwindow.append(oligopos)
            elif inwindow == True:
                if oligopos in sigoligopos: #if we are in a window and the next oligo is also sig, just extend the window
                    oligoposinwindow.append(oligopos)
                    continue
                elif oligopos not in sigoligopos:
                    findasigoligoaftergap = False #are we able to find a sig oligo within 10 oligos of gap start
                    for i in range(1, 11):
                        doesthisgappass = False #does this gap fit the requirements of x sig oligos after a gap of x
                        try:
                            oligoinquestion = alloligopos[idx + i]
                            if oligoinquestion in sigoligopos:# if we found a gap that is < 10
                                findasigoligoaftergap = True
                                gapendidx = alloligopos.index(oligoinquestion)
                                #All oligos from gapidx up to (but not including) gapidx + i must be sig in order for window to continue
                                postgapsigoligos = [] #booleans for the sig-ness of the required post-gap oligos
                                doesthisgappass = False #is this gap valid? Does it have the required number of consecutive sig oligos after the gap?
                                for j in range(0, i):
                                    try:
                                        x = alloligopos[gapendidx + j]
                                        if x in sigoligopos:
                                            postgapsigoligos.append(True)
                                        else:
                                            postgapsigoligos.append(False)
                                    except IndexError:
                                        postgapsigoligos.append(False)
                                if all(postgapsigoligos): #are all items in postgapsigoligos True
                                    doesthisgappass = True
                                    break
                                else:
                                    doesthisgappass = False

                        except IndexError:
                            inwindow = False
                            doesthisgappass = False
                            break

                        if doesthisgappass == True:
                            currentidx = idx + i + i
                            #Append all oligos from idx up to (but not including) currentidx to oligoposinwindow
                            for k in range(idx, currentidx):
                                oligoposinwindow.append(alloligopos[k])
                            inwindow = True
                            break

                    #If we didn't find a sig oligo within 10 positions of the gapstart
                    if findasigoligoaftergap == False:
                        inwindow = False

        #Turn this list of positions in a window into a list of tuples of windowstart/windowstop
        #https://stackoverflow.com/questions/2154249/identify-groups-of-continuous-numbers-in-a-list
        windows = [] #tuples of (windowstart, windowstop)
        for k, g in groupby(enumerate(oligoposinwindow), lambda x:x[0] - x[1]):
            group = list(map(itemgetter(1), g))
            windows.append((group[0], group[-1]))

        #Merge windowA and windowB if they are less than min(windowAlength, windowBlength) + 2 apart
        mergedwindows = [] #list of positions in merged windows
        for windowA in windows:
            windowAstart = windowA[0]
            windowAend = windowA[1]
            windowAlength = (windowAend - windowAstart) + 1
            wasmerged = False
            for windowB in windows:
                windowBstart = windowB[0]
                windowBend = windowB[1]
                windowBlength = (windowBend - windowBstart) + 1
                allowabledistance = min([windowAlength, windowBlength]) + 2
                if abs(windowBstart - windowAend) <= allowabledistance or abs(windowAstart - windowBend) <= allowabledistance:
                    mergedwindow = (min(windowAstart, windowBstart), max(windowAend, windowBend))
                    for i in range(mergedwindow[0], mergedwindow[1] + 1):
                        mergedwindows.append(i)
                    wasmerged = True

            #if this window was not merged with anything, just put the original window into mergedwindows
            if not wasmerged:
                for i in range(windowAstart, windowAend + 1):
                    mergedwindows.append(i)

        #remove duplicates and sort mergedwindows
        mergedwindows = sorted(list(set(mergedwindows)))

        #Break consective windows into chunks as above
        windows = [] #tuples of (windowstart, windowstop)
        for k, g in groupby(enumerate(mergedwindows), lambda x:x[0] - x[1]):
            group = list(map(itemgetter(1), g))
            windows.append((group[0], group[-1]))

        #Filter for windows that are at least windowlengthminimum long
        windows = [window for window in windows if ((window[1] - window[0]) + 1) >= windowlengthminimum]

        #Add data about windows to dictionary
        positionsinawindow = []
        enrichedwindows[gene] = []
        for window in windows:
            enrichedwindows[gene].append(window)
            for position in range(window[0], window[1] + 1):
                positionsinawindow.append(position)

        #Now add a new column to the df that tells whether or not that position is in a window
        inwindow = []
        for i in alloligopos:
            if i in positionsinawindow:
                inwindow.append('yes')
            else:
                inwindow.append('no')

        #Add column to df
        genedf = genedf.assign(inwindow = inwindow)
        genedfs.append(genedf)

    #Concatentate all genedfs back together
    allgenedfs = pd.concat(genedfs, axis = 0)

    return enrichedwindows, allgenedfs

def getminimalseqs(enrichedwindows, oligogff, cushion):
    #Given a set of oligo windows provided by definewindows(), get the minimal sequence
    #within those windows associated with localization. If windowmode == 'intersect', these will be defined as the sequence
    #in common to all oligos within the window. If windowmode == 'union', these will be defined as the sequence
    #in any oligo within the window

    #enrichedwindows has the following structure:
    #{genename : [(first oligo in window1, last oligo in window1), (first oligo in window2, last oligo in window2)]}
    #In this dictionary, oligos are defined by their position within the UTR, starting with 1
    #This designations correspond to the numbers after the ENS id and '.' in the oligo gff

    minimalseqs = [] #[[chrm, 'mm10', 'minimalseq', start, stop, '.', strand, attributes]]
    windowoligosgff = []

    print('Indexing gff...')
    gff_fn = oligogff
    db_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge', verbose = True)

    db = gffutils.FeatureDB(db_fn)
    print('Done indexing!')

    for gene in enrichedwindows:
        windowcounter = 0
        for window in enrichedwindows[gene]:
            windowcounter +=1
            coverednt = [] #[[all nt covered by oligo1], [all nt covered by oligo2], ...]
            firstoligo = window[0]
            lastoligo = window[1]
            utrs = db.features_of_type('UTR')
            for utr in utrs:
                genename = utr.attributes['gene_name'][0]
                if genename == gene:
                    chrm = str(utr.chrom)
                    strand = str(utr.strand)
                    ensid = str(utr.id).split('.')[0]
                    for oligo in db.children(utr, featuretype = 'oligo', level = 1):
                        oligonumber = int(str(oligo.id).split('.')[1])
                        #if this is one of the oligos we are after
                        if oligonumber >= firstoligo and oligonumber <= lastoligo:
                            #is this a oneexon oligo?
                            if oligo.attributes['oligo_type'][0] == 'regular_oneexon':
                                coverednt.append(list(range(oligo.start, oligo.end + 1)))
                                windowoligosgff.append([str(oligo.chrom), 'sigWindowOligos', 'oligo', str(oligo.start), str(oligo.stop), '.', oligo.strand, '.', 'ID={0};gene_name={1};oligo_type={2}'.format(str(oligo.id), str(oligo.attributes['gene_name'][0]), 'regular_oneexon')])
                            elif oligo.attributes['oligo_type'][0] == 'junction':
                                thisoligosnt = []
                                windowoligosgff.append([str(oligo.chrom), 'sigWindowOligos', 'oligo', str(oligo.start), str(oligo.stop), '.', oligo.strand, '.', 'ID={0};gene_name={1};oligo_type={2}'.format(str(oligo.id), str(oligo.attributes['gene_name'][0]), 'junction')])
                                for junctionpiece in db.children(oligo, featuretype = 'junctionpiece'):
                                    windowoligosgff.append([str(oligo.chrom), 'sigWindowOligos', 'junctionpiece', str(junctionpiece.start), str(junctionpiece.stop), '.', oligo.strand, '.', 'ID={0};gene_name={1};oligo_type={2};Parent={3}'.format(str(junctionpiece.id), str(junctionpiece.attributes['gene_name'][0]), 'junction', str(junctionpiece.attributes['Parent'][0]))])
                                    junctionpiecent = list(range(junctionpiece.start, junctionpiece.end + 1))
                                    thisoligosnt += junctionpiecent
                                coverednt.append(thisoligosnt)

            #now get the nt in the window
            if gene == 'Cplx2':
                windowmode = 'union'
            else:
                windowmode = 'union'
            if windowmode == 'intersect':
                commonnt = set(coverednt[0])
                for x in coverednt[1:]:
                    commonnt.intersection_update(x)
                commonnt = sorted(list(commonnt))
            elif windowmode == 'union':
                setlist = [set(x) for x in coverednt]
                unionnt = set().union(*setlist)
                commonnt = sorted(list(unionnt))

            #Does this minimal element span an intron?
            #Turn this list of positions in a window into a list of tuples of windowstart/windowstop
            #https://stackoverflow.com/questions/2154249/identify-groups-of-continuous-numbers-in-a-list
            #length of minimalelementwindows is the number of exons contained in this minimal element
            minimalelementwindows = [] #tuples of (windowstart, windowstop)
            for k, g in groupby(enumerate(commonnt), lambda x:x[0] - x[1]):
                group = list(map(itemgetter(1), g))
                minimalelementwindows.append((group[0], group[-1]))

            #if this whole minimal element is contained within one exon
            if len(minimalelementwindows) == 1:
                minimalseq = [chrm, 'mm10', 'minimalseq', str(minimalelementwindows[0][0] - cushion), str(minimalelementwindows[-1][1] + cushion), '.', strand, '.', 'ID={0}.minimalseq{1};gene_name={2};element_type={3}'.format(ensid, windowcounter, gene, 'singleexon')]
                minimalseqs.append(minimalseq)
            #if it spans multiple exons
            elif len(minimalelementwindows) > 1:
                minimalelementexoncounter = 0
                wholewindow = [chrm, 'mm10', 'minimalseq', str(minimalelementwindows[0][0] - cushion), str(minimalelementwindows[-1][1] + cushion), '.', strand, '.', 'ID={0}.minimalseq{1};gene_name={2};element_type={3}'.format(ensid, windowcounter, gene, 'multiexon')]
                minimalseqs.append(wholewindow)
                for x in minimalelementwindows:
                    minimalelementexoncounter +=1
                    minimalseq = [chrm, 'mm10', 'minimalseqexon', str(x[0]), str(x[1]), '.', strand, '.', 'ID={0}.minimalseq{1}.exon{2};gene_name={3};element_type={4};Parent={5}'.format(ensid, windowcounter, minimalelementexoncounter, gene, 'multiexon', ensid + '.minimalseq' + windowcounter)]
                    minimalseqs.append(minimalseq)

    return minimalseqs, windowoligosgff

def makenonwindowgff(oligogff, windowgff):
    #Given a gff of oligo positions, first get rid of any sequence that contains CDS. This happens because we included sequence upstream
    #of the start codon to get good coverage of the beginning of the UTR. As of now, this removal is done by hand.
    #Then, make a gff that contains UTR regions that are covered by oligos but NOT by a significant window. This is in preparation for looking a sequence content, etc.
    #of UTR regions in significant windows vs. not in significant windows.
    #Sig windows are minimal seqs as defined by getminimalseqs

    #TODO: get these coordinates from mm10 gff
    firstfullyUTRoligo = {} #{genename : coordinate of first UTR nt in UTRs covered by oligos}
    firstfullyUTRoligo['Trak2'] = 58903503
    firstfullyUTRoligo['Fnbp4'] = 90779652
    firstfullyUTRoligo['Trp53inp2'] = 155386793
    firstfullyUTRoligo['Akap12'] = 4357932
    firstfullyUTRoligo['Gdf11'] = 128885105
    firstfullyUTRoligo['Net1'] = 3884069
    firstfullyUTRoligo['Cplx2'] = 54379710
    firstfullyUTRoligo['Paxbp1'] = 91015060
    firstfullyUTRoligo['Kif5b'] = 6208204
    firstfullyUTRoligo['Afap1l1'] = 61731286
    firstfullyUTRoligo['Malat1'] = 5802671
    firstfullyUTRoligo['Cdc42bpg'] = 6324574
    firstfullyUTRoligo['Ogt'] = 101682331
    firstfullyUTRoligo['Rab13'] = 90225584

    print('Indexing gff...')
    gff_fn = oligogff
    oligodb_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(oligodb_fn) == False:
        gffutils.create_db(gff_fn, oligodb_fn, merge_strategy = 'merge', verbose = True)

    oligodb = gffutils.FeatureDB(oligodb_fn)
    print('Done indexing!')

    print('Indexing gff...')
    gff_fn = windowgff
    windowdb_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(windowdb_fn) == False:
        gffutils.create_db(gff_fn, windowdb_fn, merge_strategy = 'merge', verbose = True)

    windowdb = gffutils.FeatureDB(windowdb_fn)
    print('Done indexing!')

    #Get nucelotides that are in a minimal seq window
    minimalseqnt = {} #{gene : [nt in minimalseq]}
    minimalseqs = windowdb.features_of_type('minimalseq')
    for minimalseq in minimalseqs:
        gene = minimalseq.attributes['gene_name'][0]
        if gene not in minimalseqnt:
            minimalseqnt[gene] = []
        nt = list(range(minimalseq.start, minimalseq.end + 1))
        minimalseqnt[gene] += nt

    oligononwindownt = {}  #{gene : [nt that are not in sig window]}
    genename2ensid = {} #{gene_name : ensid}

    oligos = oligodb.features_of_type('oligo')
    for oligo in oligos:
        gene = oligo.attributes['gene_name'][0]
        if gene not in oligononwindownt:
            oligononwindownt[gene] = []
        strand = oligo.strand
        utrboundary = firstfullyUTRoligo[gene]
        nt = list(range(oligo.start, oligo.end + 1))
        #Get rid of any nt outside of nt boundary
        if strand == '+':
            nt = [x for x in nt if x >= utrboundary]
        elif strand == '-':
            nt = [x for x in nt if x <= utrboundary]
        #Get rid of any nt in a sig window
        try:
            windownt = minimalseqnt[gene]
        except KeyError: #this gene wasn't in minimalseqgff (probably because it didn't have a minimal seq)
            windownt = []
        nt = [x for x in nt if x not in windownt]
        oligononwindownt[gene] += nt

        ensid = str(oligo.id).split('.')[0]
        genename2ensid[gene] = ensid

    #Go through oligononwindownt and break it up into contiguous chunks
    oligononwindownt_chunks = {} #{gene : [[chunk1start, chunk1stop], [chunk2start, chunk2stop], ...]}
    #Break consective windows into chunks as above
    for gene in oligononwindownt:
        nt = sorted(list(set(oligononwindownt[gene])))
        windows = [] #tuples of (windowstart, windowstop)
        for k, g in groupby(enumerate(nt), lambda x:x[0] - x[1]):
            group = list(map(itemgetter(1), g))
            windows.append((group[0], group[-1]))
        oligononwindownt_chunks[gene] = windows

    #Write nonwindownt to gff
    with open('nonminimalseqs.gff', 'w') as outfh:
        for gene in oligononwindownt_chunks:
            ensid = genename2ensid[gene]
            utrensid = genename2ensid[gene] + '.UTR'
            chrm = oligodb[utrensid].chrom
            strand = oligodb[utrensid].strand
            spanstart = str(oligononwindownt_chunks[gene][0][0])
            spanend = str(oligononwindownt_chunks[gene][-1][1])
            outfh.write(('\t').join([chrm, 'mm10', 'nonminimalseqspan', spanstart, spanend, '.', strand, '.', 'ID={0}.nonminimalseqspan;gene_name={1}'.format(ensid, gene)])+ '\n')
            chunkcounter = 0
            for chunk in oligononwindownt_chunks[gene]:
                chunkcounter +=1
                chunkstart = str(chunk[0])
                chunkend = str(chunk[1])

                outfh.write(('\t').join([chrm, 'mm10', 'nonminimalseqchunk', chunkstart, chunkend, '.', strand, '.', 'ID={0}.nonminimalseqchunk{1};gene_name={2};Parent={3}.nonminimalseqspan'.format(ensid, chunkcounter, gene, ensid)]) + '\n')

    os.remove(oligodb_fn)
    os.remove(windowdb_fn)

def gff2seq(gff, feature, genomefasta, outfilename):
    #Given a gff and the featurename to look for, output a fasta

    print('Indexing gff...')
    gff_fn = gff
    db_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge', verbose = True)

    db = gffutils.FeatureDB(db_fn)
    print('Done indexing!')

    with open(outfilename, 'w') as outfh:
        with gzip.open(genomefasta, 'rt') as genomefh:
            print('Indexing genome...')
            seq_dict = SeqIO.to_dict(SeqIO.parse(genomefh, 'fasta'))
            print('Done!')
            for f in db.features_of_type(feature):
                chrm = str(f.chrom)
                start = f.start
                end = f.end
                strand = f.strand
                seqid = str(f.id) + '_' + str(f.attributes['gene_name'][0])
                outfh.write('>' + seqid + '\n')
                if strand == '+':
                    seq = seq_dict[chrm].seq[start - 1 : end].upper()
                elif strand == '-':
                    seq = seq_dict[chrm].seq[start - 1 : end].reverse_complement().upper()
                outfh.write(str(seq) + '\n')

    os.remove(db_fn)

def getntcontent(fasta, outfile):
    #Get nt content of sequences in fasta
    with open(fasta, 'r') as infh, open(outfile, 'w') as outfh:
        outfh.write(('\t').join(['seq', 'length', 'A', 'U', 'G', 'C']) + '\n')
        for record in SeqIO.parse(fasta, 'fasta'):
            seqid = str(record.id)
            seq = str(record.seq)
            a = round(seq.count('A') / len(seq), 4)
            c = round(seq.count('C') / len(seq), 4)
            u = round(seq.count('T') / len(seq), 4)
            g = round(seq.count('G') / len(seq), 4)
            length = str(len(seq))
            outfh.write(('\t').join([seqid, length, str(a), str(u), str(g), str(c)]) + '\n')

def getdintcontent(fasta, outfile):
    #Get dint content of sequences in fasta
    #Get all dinucleotides
    bases = 'ATCG'
    dints = []
    for dint in product(bases, repeat = 3):
        dints.append(''.join(dint))
    dintwithu = [x.replace('T', 'U') for x in dints]
    with open(fasta, 'r') as infh, open(outfile, 'w') as outfh:
        outfh.write(('\t').join(['seq', 'length'] + dintwithu) + '\n')
        for record in SeqIO.parse(fasta, 'fasta'):
            dintfreqs = []
            seqid = str(record.id)
            genename = seqid.split('|')[1]
            if genename == 'Cplx2':
                continue
            seq = str(record.seq)
            length = str(len(seq))
            for dint in dints:
                count = seq.count(dint)
                possibledint = len(seq) - 2
                freq = round(count / possibledint, 4)
                dintfreqs.append(freq)
            dintfreqs = [str(x) for x in dintfreqs]
            outfh.write(('\t').join([seqid, length] + dintfreqs) + '\n')

def makeoligoclassfastas(deseqtable, oligofasta):
    #Given a DESeq output table and a fasta file of all oligo sequences,
    #output 3 fastas, one of neurite oligos, one of soma oligos, and one of soma oligos

    oligocategories = {} #{oligoID : neurite/soma/nonenriched}
    with open(deseqtable, 'r') as infh:
        for line in infh:
            line = line.strip().split('\t')
            if line[0] == 'ensid':
                #skip header
                continue
            #Fix oligo ID so it matches with fasta
            oligoID = line[0] + '|' + line[1]
            sig = line[8]
            if sig == 'neurite':
                oligocategories[oligoID] = 'neurite'
            elif sig == 'soma':
                oligocategories[oligoID] = 'soma'
            elif sig == 'no':
                oligocategories[oligoID] = 'nonenriched'

    with open(oligofasta, 'r') as fastafh, open('neuriteoligos.fa', 'w') as neuritefh, open('somaoligos.fa', 'w') as somafh, open('nonenrichedoligos.fa', 'w') as nonenrichedfh:
        for record in SeqIO.parse(oligofasta, 'fasta'):
            try:
                oligocategory = oligocategories[str(record.id)]
            except KeyError:
                #not all oligos are in the deseq output table
                continue
            if oligocategory == 'neurite':
                neuritefh.write('>' + str(record.id) + '\n' + str(record.seq) + '\n')
            elif oligocategory == 'soma':
                somafh.write('>' + str(record.id) + '\n' + str(record.seq) + '\n')
            elif oligocategory == 'nonenriched':
                nonenrichedfh.write('>' + str(record.id) + '\n' + str(record.seq) + '\n')

def oligophastconsmean(gff, phastconsbed, outfile):
    #Get the mean phastcons score for all nt in an oligo
    gff_fn = gff
    db_fn = os.path.basename(gff_fn) + '.db'

    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn)

    db = gffutils.FeatureDB(db_fn)

    phastconsmeans = {} # {oligoID : mean phastcons score}
    interrogated_oligos = 0
    interrogated_bps = 0
    oligos_with_scores = 0
    bps_with_scores = 0
    phastconstabix = pysam.Tabixfile(phastconsbed)
    oligos = db.features_of_type('oligo')

    for oligo in oligos:
        oligolength = 0
        phastconsscores = []
        interrogated_oligos +=1
        if interrogated_oligos % 100 == 0:
            sys.stderr.write('Interrogating oligo {0}\n'.format(interrogated_oligos))

        if oligo.attributes['oligo_type'][0] == 'regular_oneexon':
            oligolength += int(oligo.stop - oligo.start)
            for bed in phastconstabix.fetch(str(oligo.chrom), oligo.start, oligo.end, parser = pysam.asTuple()):
                score = bed[4]
                phastconsscores.append(float(score))
        elif oligo.attributes['oligo_type'][0] == 'junction':
            for junctionpiece in db.children(oligo, featuretype = 'junctionpiece', level = 1):
                oligolength += int(junctionpiece.stop - junctionpiece.start)
                for bed in phastconstabix.fetch(str(junctionpiece.chrom), junctionpiece.start, junctionpiece.end, parser = pysam.asTuple()):
                    score = bed[4]
                    phastconsscores.append(float(score))

        if len(phastconsscores) > 0:
            oligos_with_scores +=1
            phastconsmeans[str(oligo.id)] = (sum(phastconsscores) / float(len(phastconsscores)))

        scored_bps = len(phastconsscores)
        interrogated_bps += oligolength
        bps_with_scores += scored_bps

    percent_bp_with_scores = (bps_with_scores / float(interrogated_bps))*100

    sys.stderr.write('Searched {0} oligos and got PhastCons scores for {1} of them. Overall, scores were obtained for {2}% of all interrogated bps.\n'.format(interrogated_oligos, oligos_with_scores, percent_bp_with_scores))

    os.remove(db_fn)

    with open(outfile, 'w') as outfh:
        for oligo in phastconsmeans:
            outfh.write(('\t').join([str(oligo), str(phastconsmeans[oligo])]) + '\n')

def getAGwindows_oligos(oligogff, genomefasta, outfile):
    #Get smoothed windows of A/G content across UTRs
    #This one is meant to take in an oligogff

    gff_fn = oligogff
    db_fn = os.path.basename(gff_fn) + '.db'

    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn)

    db = gffutils.FeatureDB(db_fn)
    utrs = db.features_of_type('UTR')

    with gzip.open(genomefasta, 'rt') as genomefh:
        print('Indexing genome...')
        seq_dict = SeqIO.to_dict(SeqIO.parse(genomefh, 'fasta'))
        print('Done!')

    with open(outfile, 'w') as outfh:
        outfh.write(('\t').join(['genename', 'position', 'agfrac']) + '\n')
        for utr in utrs:
            chrm = str(utr.chrom)
            strand = utr.strand
            genename = utr.attributes['gene_name'][0]
            for nt in range(utr.start, utr.end):
                #20 nt window
                windowstart = nt - 10
                windowend = nt + 10
                if strand == '+':
                    seq = str(seq_dict[chrm].seq[windowstart:windowend + 1]).upper()
                elif strand == '-':
                    seq = str(seq_dict[chrm].seq[windowstart:windowend + 1].reverse_complement()).upper()
                agfrac = round((seq.count('A') + seq.count('G')) / len(seq), 4)
                outfh.write(('\t').join([genename, str(nt), str(agfrac)]) + '\n')

def getAGwindows_allUTRs(UTRgff, genomefasta, outfile):
    #Get smoothed windows of A/G content across UTRs
    #This one is meant to take in a longest UTRs gff

    gff_fn = UTRgff
    db_fn = os.path.basename(gff_fn) + '.db'

    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge')

    db = gffutils.FeatureDB(db_fn)
    utrs = db.features_of_type('UTR3')

    with gzip.open(genomefasta, 'rt') as genomefh:
        print('Indexing genome...')
        seq_dict = SeqIO.to_dict(SeqIO.parse(genomefh, 'fasta'))
        print('Done!')

    with open(outfile, 'w') as outfh:
        outfh.write(('\t').join(['genename', 'chrm', 'startcoord', 'endcoord', 'metaposition', 'agfrac']) + '\n')
        for utr in utrs:
            chrm = str(utr.chrom)
            strand = utr.strand
            genename = utr.attributes['Parent'][0][5:].split('.')[0]
            if strand == '+':
                exoniccoords = []
                seq = ''
                #Get exonic coords and seqs
                for exon in db.children(utr, featuretype = 'exon', order_by = 'start'):
                    exoniccoords += list(range(exon.start, exon.end + 1))
                    exonseq = seq_dict[chrm].seq[exon.start : exon.end + 1].upper()
                    seq += exonseq

            elif strand == '-':
                exoniccoords = []
                seq = ''
                #Get exonic coords and seqs
                for exon in db.children(utr, featuretype = 'exon', order_by = 'start', reverse = True):
                    exoniccoords += list(reversed(list(range(exon.start, exon.end + 1))))
                    exonseq = seq_dict[chrm].seq[exon.start : exon.end + 1].reverse_complement().upper()
                    seq += exonseq

            if len(seq) < 100:
                continue

            for x in list(range(len(seq))):
                if x % 5 != 0:
                    continue
                #window size = 100
                #step size = 5
                if x < 50:
                    windowboundaries = [0, x + 50]
                elif len(seq) - x < 50:
                    windowboundaries = [x, len(seq)]
                else:
                    windowboundaries = [x - 50 , x + 50]

                startcoord = exoniccoords[windowboundaries[0]]
                endcoord = exoniccoords[windowboundaries[1] - 1]
                metaposition = round(x / float(len(seq)), 3)
                windowseq = seq[windowboundaries[0] : windowboundaries[1]]
                agfrac = (windowseq.count('A') + windowseq.count('G')) / len(windowseq)
                outfh.write(('\t').join([genename, chrm, str(startcoord), str(endcoord), str(metaposition), str(agfrac)]) + '\n')

'''
#Relate oligo coords to genomecoords
#needs probegff
oligocoorddf = simpleroligos2genome(sys.argv[1])
print(oligocoorddf.head())

#Define enriched windows of oligos
enrichedwindows, df = definewindows(sys.argv[2], 10)
#print(df.head())
print(enrichedwindows)

#Join oligocoord and window dfs
df = pd.merge(oligocoorddf, df, how = 'inner', on = ['genename', 'oligopos'])
print(enrichedwindows)
df.to_csv(path_or_buf = os.path.abspath(sys.argv[2]) + '.sigoligowindows', sep = '\t', na_rep = 'NA', index = False, header = True)

#Define seqs of interest based on enriched windows
minimalseqs, windowoligosgff = getminimalseqs(enrichedwindows, sys.argv[1], 0)
with open('minimalseqs.gff', 'w') as outfh:
    for minimalseq in minimalseqs:
        print(minimalseq[8], int(minimalseq[4]) - int(minimalseq[3]))
        outfh.write(('\t').join(minimalseq) + '\n')

with open('windowoligos.gff', 'w') as outfh:
    for windowoligo in windowoligosgff:
        outfh.write(('\t').join(windowoligo) + '\n')

#Make a gff for UTR regions NOT in a minimal seq
#oligogff, minimalseqgff
makenonwindowgff(sys.argv[1], 'minimalseqs.gff')

#Write fastas for minimalseq and nonminimalseq gffs
gff2seq('minimalseqs.gff', 'minimalseq', sys.argv[3], 'minimalseqs.fa')
gff2seq('nonminimalseqs.gff', 'nonminimalseqchunk', sys.argv[3], 'nonminimalseqs.fa')
'''

#getntcontent('minimalseqs.fa', 'minimalseqntcontent.txt')
#getntcontent('nonminimalseqs.fa', 'nonminimalseqntcontent.txt')
#getntcontent('windowseqs.fa', 'windowseqntcontent.txt')
#getntcontent('nonwindowseqs.fa', 'nonwindowseqntcontent.txt')
#getntcontent('neuriteoligos.fa', 'neuriteoligontcontent.txt')
#getntcontent('somaoligos.fa', 'somaoligontcontent.txt')
#getntcontent('nonenrichedoligos.fa', 'nonenrichedoligontcontent.txt')

#getdintcontent('neuriteoligos.fa', 'neuriteoligotrintcontent.nocplx2.txt')
#getdintcontent('somaoligos.fa', 'somaoligotrintcontent.nocplx2.txt')
#getdintcontent('nonenrichedoligos.fa', 'nonenrichedoligotrintcontent.nocplx2.txt')
'''
#Write fastas of oligo sequences
makeoligoclassfastas(sys.argv[1], sys.argv[2])

#Get oligo conservation
#oligogff, phastconsbed, outfile
oligophastconsmean(sys.argv[1], sys.argv[2], sys.argv[3])

'''
#Get AG content across UTRs
#oligogff, genomefasta, outfile
#getAGwindows(sys.argv[1], sys.argv[2], sys.argv[3])

getAGwindows_allUTRs(sys.argv[1], sys.argv[2], sys.argv[3])

#TODO
#generate random regions from non-minimal seq gff
#sliding G/A content window across UTR -- for each nt in the UTR (as defined by oligos.gff), calculate sliding average GAcontent


from methods.BandB.ptype.utils import normalize_log_probs, log_weighted_sum_probs, log_weighted_sum_probs_check, print_to_file, log_weighted_sum_normalize_probs
from scipy import optimize
from methods.BandB.ptype.Config import Config
import numpy as np
import time
import sys
from scipy.stats import norm

Inf = np.Inf

def vecnorm(x, ord=2):
    if ord == Inf:
        return np.amax(np.abs(x))
    elif ord == -Inf:
        return np.amin(np.abs(x))
    else:
        return np.sum(np.abs(x)**ord, axis=0)**(1.0 / ord)

LOG_EPS = -1e150

class PtypeModel:
    TYPE_INDEX = 0
    MISSING_INDEX = 1
    ANOMALIES_INDEX = 2
    LLHOOD_TYPE_START_INDEX = 2

    def __init__(self, _experiment_config, _data_frame=None, _PI=[0.98, 0.01, 0.01]):
        self.experiment_config = _experiment_config
        self.data = _data_frame
        self.PI = _PI  # weight of pi variable
        self.F_training = []
        self.F_validation = []
        self.data_frames = None
        self.labels = None
        self.types = None

    ###################### MAIN METHODS #######################
    def run_inference(self, logP, counts):
        sys.stdout.flush()
        # Constants
        I, J = logP.shape               # I: num of rows in a data column.
                                        # J: num of data types including missing and catch-all
        K = J - 2                       # K: num of possible column data types (excluding missing and catch-all)

        # Initializations
        pi = [self.PI for j in range(K)]          # mixture weights of row types

        # Inference
        p_t = []                        # p_t: posterior probability distribution of column types
        p_z = np.zeros((I,K,3))         # p_z: posterior probability distribution of row types

        counts_array = np.reshape(counts, newshape=(len(counts),))

        # Iterates for each possible column type
        for j in range(K):

            # Sum of weighted likelihoods (log-domain)
            p_t.append((counts_array * log_weighted_sum_probs(pi[j][0], logP[:,j+self.LLHOOD_TYPE_START_INDEX],
                                                              pi[j][1], logP[:,self.MISSING_INDEX-1],
                                                              pi[j][2], logP[:, self.ANOMALIES_INDEX - 1])).sum())

            # Calculates posterior cell probabilities

            # p_z[:, j, self.TYPE_INDEX] = np.log() +
            # p_z[:, j, self.MISSING_INDEX] = np.log(pi[j][1]) + logP[:,self.MISSING_INDEX-1]
            # p_z[:, j, self.ANOMALIES_INDEX] = np.log(pi[j][2]) + logP[:, self.ANOMALIES_INDEX - 1]

            # Normalizes
            x1, x2, x3, log_mx, sm = log_weighted_sum_normalize_probs(pi[j][0], logP[:,j+self.LLHOOD_TYPE_START_INDEX], pi[j][1], logP[:,self.MISSING_INDEX-1], pi[j][2], logP[:, self.ANOMALIES_INDEX - 1])

            p_z[:, j, 0] = np.exp(x1 - log_mx - np.log(sm))
            p_z[:, j, 1] = np.exp(x2 - log_mx - np.log(sm))
            p_z[:, j, 2] = np.exp(x3 - log_mx - np.log(sm))
            p_z[:,j,:] = p_z[:,j,:]/p_z[:,j,:].sum(axis=1)[:, np.newaxis]

        self.p_t = normalize_log_probs(np.reshape(p_t, newshape=(len(p_t),)))
        self.p_z = p_z

    def calculate_likelihoods(self, logP, counts):
        sys.stdout.flush()
        # Constants
        I, J = logP.shape               # I: num of rows in a data column.
                                        # J: num of data types including missing and catch-all
        K = J - 2                       # K: num of possible column data types (excluding missing and catch-all)

        # Initializations
        pi = [self.PI for j in range(K)]          # mixture weights of row types

        # Inference
        p_t = []                        # p_t: posterior probability distribution of column types
        p_z = np.zeros((I,K,3))         # p_z: posterior probability distribution of row types

        counts_array = np.reshape(counts, newshape=(len(counts),))

        # Iterates for each possible column type
        for j in range(K):

            # Sum of weighted likelihoods (log-domain)
            p_t.append((counts_array * log_weighted_sum_probs(pi[j][0], logP[:,j+self.LLHOOD_TYPE_START_INDEX],
                                                              pi[j][1], logP[:,self.MISSING_INDEX-1],
                                                              pi[j][2], logP[:, self.ANOMALIES_INDEX - 1])).sum())
        self.p_t = np.array(p_t)

    def train_all_z_multiple_dfs(self, runner):
        sys.stdout.flush()

        # Initializations
        self.J = len(runner.machines)                # J: num of data types including missing and anomaly.
        self.K = self.J - 2                          # K: num of possible column data types (excluding missing and anomaly)
        self.pi = [self.PI for j in range(self.K)]   # mixture weights of row types

        self.current_runner = runner

        w_j_z = self.get_all_parameters_z(runner)

        # Find new values using Conjugate Gradient method
        res = optimize.minimize(self.f_cols, w_j_z, jac=self.g_cols, method='CG', options={'disp':True, })
        if res.success:
            runner, temp = self.set_all_probabilities_z(runner, res.x, normalize=True)
        else:
            runner, temp = self.set_all_probabilities_z(runner, res.x, normalize=True)

        return runner

    def train_all_z_multiple_dfs_new(self, runner):
        sys.stdout.flush()

        # Initializations
        self.J = len(runner.machines)                # J: num of data types including missing and anomaly.
        self.K = self.J - 2                          # K: num of possible column data types (excluding missing and anomaly)
        self.pi = [self.PI for j in range(self.K)]   # mixture weights of row types

        self.current_runner = runner

        w_j_z = self.get_all_parameters_z(runner)

        # Find new values using Conjugate Gradient method
        w_j_z, j = self.conjugate_gradient(w_j_z)

        runner, temp = self.set_all_probabilities_z(runner, w_j_z, normalize=True)

        return runner

    def conjugate_gradient(self, w, J=10, gtol=1e-5):
        d, g = [], []

        gnorm = gtol + 1
        j = 0
        while (gnorm > gtol) and (j < J):
            if j == 0:
                g.append(self.g_cols(w))
                d.append(-g[j])

            res = optimize.line_search(self.f_cols, self.g_cols, w, d[j], g[j], self.f_cols(w))
            if res[0] is None:
                return w,j
            else:
                alpha = res[0]
                w = w + alpha * d[j]

                g.append(self.g_cols(w))
                gnorm = vecnorm(g[j + 1], ord=np.Inf)

                beta_j = max(0, np.dot(g[j + 1].T, g[j + 1] - g[j]) / np.dot(g[j], g[j]))  # eq. 7.74 Polak-Ribiere
                d.append(-g[j + 1] + beta_j * d[j])  # eq.7.67

                j += 1

        return w, j

    def f_col(self, i_, column_name, y_i,):
        [temp_x, counts_array] = self.dfs_unique_vals_counts[i_][column_name]
        logP = np.array([self.all_probs[str(x_i)] for x_i in temp_x])
        q = []
        for k in range(self.K):
            q.append((counts_array * log_weighted_sum_probs(self.pi[k][0], logP[:, k + self.LLHOOD_TYPE_START_INDEX],
                                                            self.pi[k][1], logP[:, self.MISSING_INDEX - 1],
                                                            self.pi[k][2], logP[:, self.ANOMALIES_INDEX - 1])).sum())
        temp = normalize_log_probs(q)[y_i]
        if temp == 0:
            error = +800./len(counts_array)
        else:
            error = - np.log(temp)/len(counts_array)

        # result_dict[process_id] = error
        return error

    def f_cols(self, w_j_z):
        # f: the objective function to minimize. (it is equal to - \sum_{all columns} log p(t=k|X) where k is the correct column type.)
        # print_to_file('f_cols is called')
        time_init = time.time()

        # Set params: init-transition-final
        runner, temp_w_j_z = self.set_all_probabilities_z(self.current_runner, w_j_z)

        # Generate probabilities
        self.all_probs = runner.generate_machine_probabilities(self.unique_vals)

        error = 0.
        for i, (data_frame, labels) in enumerate(zip(self.data_frames, self.labels)):
            for j, column_name in enumerate(list(data_frame.columns)):
                error += self.f_col(str(i), column_name, labels[j] - 1)
        # print_to_file(str(time.time() - time_init))
        # print(error)
        return error

    def dp_dz(self, zs):
        temp = np.exp(zs)
        return (temp * sum(temp) - temp * temp)/(sum(temp)**2)

    def g_col(self, runner, i_, column_name, y_i):
        [temp_x, counts_array] = self.dfs_unique_vals_counts[i_][column_name]
        logP = np.array([self.all_probs[str(x_i)] for x_i in temp_x])

        set_chars = np.unique(sum([list(str(x_i)) for x_i in temp_x], []))

        # calculates posterior values of types
        q = []
        for k in range(self.K):
            q.append((counts_array * log_weighted_sum_probs(self.pi[k][0], logP[:, k + self.LLHOOD_TYPE_START_INDEX],
                                                            self.pi[k][1], logP[:, self.MISSING_INDEX - 1],
                                                            self.pi[k][2], logP[:, self.ANOMALIES_INDEX - 1])).sum())

        # calculates the gradients for initial, transition, and final probabilities. (note that it is only for non-zero probabilities at the moment.)
        g_j = []
        for t in list(self.experiment_config.types.keys()):
            t -= 1
            x_i_indices = np.where(logP[:, t + 2] != LOG_EPS)[0]
            possible_states = [state for state in runner.machines[2 + t].states if runner.machines[2 + t].I[state] != LOG_EPS]
            A = log_weighted_sum_probs(self.pi[t][0], logP[:, t + self.LLHOOD_TYPE_START_INDEX],
                                       self.pi[t][1], logP[:, self.MISSING_INDEX - 1],
                                       self.pi[t][2], logP[:, self.ANOMALIES_INDEX - 1])
            temp_gra = np.exp(self.pi[t][0] + logP[:, t + 2] - A)

            temp_g_j = []
            for state in possible_states:
                temp_g_j.append(self.gradient_initial_optimized_new(runner, state, t, temp_x[x_i_indices], q, temp_gra[x_i_indices], counts_array[x_i_indices], y_i))
            g_j = g_j + temp_g_j

            for a in runner.machines[2 + t].T:
                temp_g_j = []
                for b in runner.machines[2 + t].T[a]:
                    if str(b) not in set_chars:
                        temp_g_j = temp_g_j + [0 for i in range(len(runner.machines[2 + t].T[a][b].keys()))]
                    else:
                        for c in runner.machines[2 + t].T[a][b]:
                            temp_g_j.append(self.gradient_transition_optimized_new(runner, a, b, c, t, q, temp_x[x_i_indices], y_i, temp_gra[x_i_indices], counts_array[x_i_indices]))
                        # cs_temp = [runner.machines[2 + t].calculate_gradient_abc_new_optimized(str(x_i), b) for x_i in temp_x]
                        # cs_temp = np.reshape(cs_temp, newshape=(len(cs_temp),))
                        # temp_mult = (temp_gra * cs_temp * counts_array).sum()
                        # for c in runner.machines[2 + t].T[a][b]:
                        #     temp_g_j.append(self.gradient_transition_optimized_new(runner, a, b, c, t, q, temp_mult, y_i))
                g_j = g_j + temp_g_j

            for state in runner.machines[2 + t].F:
                if runner.machines[2 + t].F[state] != LOG_EPS:

                    g_j.append(self.gradient_final_optimized_new(runner, state, t, temp_x[x_i_indices], q, temp_gra[x_i_indices], counts_array[x_i_indices], y_i))

        # result_dict[process_id] = -np.reshape(g_j, newshape=(len(g_j),))
        # print('g', -np.reshape(g_j, newshape=(len(g_j),))/counts_array.sum())
        return -np.reshape(g_j, newshape=(len(g_j),))/counts_array.sum()

    def g_col_marginals(self, runner, i_, column_name, y_i):
        [temp_x, counts_array] = self.dfs_unique_vals_counts[i_][column_name]
        logP = np.array([self.all_probs[str(x_i)] for x_i in temp_x])

        # set_chars = np.unique(sum([list(str(x_i)) for x_i in temp_x], []))

        # calculates posterior values of types
        r = []
        for k in range(self.K):
            r.append((counts_array * log_weighted_sum_probs(self.pi[k][0], logP[:, k + self.LLHOOD_TYPE_START_INDEX],
                                                            self.pi[k][1], logP[:, self.MISSING_INDEX - 1],
                                                            self.pi[k][2], logP[:, self.ANOMALIES_INDEX - 1])).sum())

        # calculates the gradients for initial, transition, and final probabilities. (note that it is only for non-zero probabilities at the moment.)
        g_j = []
        for t in list(self.experiment_config.types.keys()):
            t -= 1
            x_i_indices = np.where(logP[:, t + 2] != LOG_EPS)[0]
            # print('g_col_marginals', t, len(x_i_indices))

            possible_states = [state for state in runner.machines[2 + t].states if runner.machines[2 + t].I[state] != LOG_EPS]
            A = log_weighted_sum_probs(self.pi[t][0], logP[:, t + self.LLHOOD_TYPE_START_INDEX],
                                       self.pi[t][1], logP[:, self.MISSING_INDEX - 1],
                                       self.pi[t][2], logP[:, self.ANOMALIES_INDEX - 1])
            temp_gra = np.exp(self.pi[t][0] + logP[:, t + 2] - A)

            # gradient for initial state parameters
            temp_g_j = []
            for state in possible_states:
                temp_g_j.append(self.gradient_initial_optimized_new(runner, state, t, temp_x[x_i_indices], r, temp_gra[x_i_indices], counts_array[x_i_indices], y_i))
            g_j = g_j + temp_g_j

            # gradient for transition parameters
            # print('generating marginals')
            if t == 1:
                marginals = {str(x_i): np.ones((len(x_i),1,1)) if p_x_i[t + 2] != LOG_EPS else np.zeros((len(x_i),1,1)) for x_i, p_x_i in zip(temp_x, logP)}
            else:
                marginals = {str(x_i): runner.machines[2 + t].run_forward_backward(str(x_i)) if p_x_i[t+2] != LOG_EPS else np.zeros((len(x_i),len(x_i))) for x_i, p_x_i in zip(temp_x, logP)}
            # print('generated marginals')
            state_indices = {}
            counter = 0
            temp_g_j = []
            for a in runner.machines[2 + t].T:
                for b in runner.machines[2 + t].T[a]:
                    for c in runner.machines[2 + t].T[a][b]:
                        state_indices[str(a) + '*' + str(b) + '*' + str(c)] = counter
                        temp_g_j.append(0)
                        counter += 1
            # print('first done')
            for x_i_index, (x_i, temp_gra_i, counts_array_i) in enumerate(zip(temp_x[x_i_indices], temp_gra[x_i_indices], counts_array[x_i_indices])):
                if logP[x_i_index, t+2] != LOG_EPS:
                    if t == 1:
                        # for a in runner.machines[2 + t].T:
                        #     temp_g_j = []
                        #     for b in runner.machines[2 + t].T[a]:
                        #         if str(b) in set_chars:
                        #             for c in runner.machines[2 + t].T[a][b]:
                        #                 temp_g_j[state_indices[a + '*' + b + '*' + c]] += self.gradient_transition_optimized_new_marginals(runner, marginals, a, b, c, t, r, str(x_i), y_i, temp_gra_i, counts_array_i)
                        common_chars = list(set(list(str(x_i))) & set(runner.machines[t+2].alphabet))
                        for common_char in common_chars:
                            common_char_ls = np.where(list(str(x_i)) == common_char)[0]
                            for l in common_char_ls:
                                indices_nonzero = np.where(marginals[str(x_i)][l] != 0.)
                                if len(indices_nonzero[0]) != 0:
                                    q_s = indices_nonzero[0]
                                    q_primes = indices_nonzero[1]
                                    for q, q_prime in zip(q_s, q_primes):
                                        temp_g_j[state_indices[str(runner.machines[t + 2].states[q]) + '*' + str(common_char) + '*' + str(
                                            runner.machines[t + 2].states[q_prime])]] += self.gradient_transition_optimized_new_marginals(runner, marginals,
                                                                                runner.machines[t + 2].states[q], common_char,
                                                                                runner.machines[t + 2].states[q_prime], t, r,
                                                                                str(x_i), y_i, temp_gra_i, counts_array_i)

                    else:
                        for l, alpha in enumerate(str(x_i)):
                            if alpha in runner.machines[t+2].alphabet:
                                indices_nonzero = np.where(marginals[str(x_i)][l] != 0.)
                                if len(indices_nonzero[0]) != 0:
                                    q_s = indices_nonzero[0]
                                    q_primes = indices_nonzero[1]
                                    for q, q_prime in zip(q_s, q_primes):
                                        temp_g_j[state_indices[str(runner.machines[t + 2].states[q]) + '*' + str(alpha) + '*' + str(runner.machines[t + 2].states[q_prime])]] += self.gradient_transition_optimized_new_marginals(runner, marginals, runner.machines[t+2].states[q], alpha, runner.machines[t+2].states[q_prime], t, r, str(x_i), y_i, temp_gra_i, counts_array_i)
            # print('transition done')
            g_j = g_j + temp_g_j

            # gradient for final-state parameters
            for state in runner.machines[2 + t].F:
                if runner.machines[2 + t].F[state] != LOG_EPS:
                    g_j.append(self.gradient_final_optimized_new(runner, state, t, temp_x[x_i_indices], r, temp_gra[x_i_indices], counts_array[x_i_indices], y_i))
            # print('final done')

        # print('g', -np.reshape(g_j, newshape=(len(g_j),))/counts_array.sum())
        return -np.reshape(g_j, newshape=(len(g_j),))/counts_array.sum()

    def g_cols(self, w_j_z):
        print_to_file('g_cols is called')
        time_init = time.time()

        # calculates the gradient vector, i.e. df/dw (=df/dz * dz/dw) where f is the object function to minimize.
        # it returns -g_j because of minimizing instead of maximizing. see the objective function.
        runner = self.current_runner

        # updates the parameters
        runner, temp_w_j_z = self.set_all_probabilities_z(runner, w_j_z)

        # generates probabilities
        # time_init2 = time.time()
        self.all_probs = runner.generate_machine_probabilities(self.unique_vals)
        # print_to_file(str(time.time() - time_init2))

        q_total = None
        counter_ = 0

        for i, (data_frame, labels) in enumerate(zip(self.data_frames, self.labels)):
            # print(i)
            for j, column_name in enumerate(list(data_frame.columns)):
                time_temp1 = time.time()
                # print(column_name)
                if counter_ == 0:
                    q_total = self.g_col_marginals(runner, str(i), str(column_name), labels[j] - 1)
                    counter_ += 1
                else:
                    q_total += self.g_col_marginals(runner, str(i), str(column_name), labels[j] - 1)

        print_to_file(str(time.time() - time_init))
        # print_to_file('grad chek is called.')
        # q_approx = self.grad_chek(w_j_z)
        # print(q_total)
        # print_to_file(q_approx)
        # print(q_total-q_approx)
        # print_to_file('gradient norm ' + str(vecnorm(q_total, ord=np.Inf)))
        # print_to_file('gradient approx norm' + str(vecnorm(q_approx, ord=np.Inf)))
        # print_to_file('gradients diff norm' + str(vecnorm(q_total - q_approx)))

        return q_total

    ###################### GRADIENT HELPERS #######################
    def scale_wrt_type(self, gradient, q, t, y_i):
        temp = normalize_log_probs(q)[t]
        return gradient * (1 - temp) if t == y_i else -1 * gradient * temp

    def gradient_initial_optimized_new(self, runner, state, t, x, q, temp, counter, y_i):
        exp_param = 1 - np.exp(runner.machines[2 + t].I[state])

        cs_temp = [runner.machines[2 + t].calculate_gradient_initial_state_optimized(str(x_i), state) for x_i in x]
        cs = np.reshape(cs_temp, newshape=(len(cs_temp),))

        gradient = (temp * counter * cs * exp_param).sum()
        return (self.scale_wrt_type(gradient, q, t, y_i))

    def gradient_transition_optimized_new(self, runner, a, b, c, t, q, x, y_i, temp_gra, counts_array):

        cs_temp = [runner.machines[2 + t].calculate_gradient_abc_new_optimized(str(x_i), a, b, c) for x_i in x]
        cs_temp = np.reshape(cs_temp, newshape=(len(cs_temp),))
        temp_mult = (temp_gra * cs_temp * counts_array).sum()

        exp_param = 1 - np.exp(runner.machines[2 + t].T[a][b][c])
        gradient = exp_param * temp_mult

        return self.scale_wrt_type(gradient, q, t, y_i)

    def gradient_transition_optimized_new_marginals(self, runner, marginals, a, b, c, t, q, x, y_i, temp_gra, counts_array):

        temp_mult = temp_gra * runner.machines[2 + t].calculate_gradient_abc_new_optimized_marginals(marginals[str(x)], str(x), a, b, c) * counts_array
        exp_param = 1 - np.exp(runner.machines[2 + t].T[a][b][c])
        gradient = exp_param * temp_mult

        return self.scale_wrt_type(gradient, q, t, y_i)

    def gradient_final_optimized_new(self, runner, final_state, t, x, q, temp, counter, y_i):
        exp_param = 1 - np.exp(runner.machines[2 + t].F[final_state])

        cs_temp = [runner.machines[2 + t].calculate_gradient_final_state_optimized(str(x_i), final_state) for x_i in x]
        cs = np.reshape(cs_temp, newshape=(len(cs_temp),))
        gradient = sum(temp * counter * cs * exp_param)

        return self.scale_wrt_type(gradient, q, t, y_i)

    ###################### HELPERS #######################
    def grad_chek(self, w_j_z):
        EPS = 1e-8

        runner = self.current_runner
        grad_approx = []

        for k in range(len(w_j_z)):
            # updates the parameters
            w_j_z[k] = w_j_z[k] + EPS
            runner, temp_w_j_z = self.set_all_probabilities_z(runner, w_j_z)
            f_fwd = self.f_cols(w_j_z)

            # updates the parameters
            w_j_z[k] = w_j_z[k] - 2 * EPS
            runner, temp_w_j_z_ignored = self.set_all_probabilities_z(runner, w_j_z)
            f_bwd = self.f_cols(w_j_z)

            grad_approx.append((f_fwd - f_bwd) / (2 * EPS))

            w_j_z = temp_w_j_z

        return np.array(grad_approx)

    ### GETTERS - SETTERS ###
    def set_data(self, data):
        self.data = data

    def set_data_multiple_dfs(self, _data_frame, _dataset_name=None, _column_names=None):
        if _dataset_name is None:
            _dataset_name = 'demo'
        _data_frame = _data_frame.rename(columns=lambda n: str(n).replace(' ', ''))

        # Check for dataframes without column names
        if _column_names is None:
            _column_names = _data_frame.columns

        # Creates a configuration object for the experiments
        config = Config(_dataset_name=_dataset_name, _column_names=_column_names, _types=self.types)

        # Ptype model for inference
        self.set_params(config, _data_frame=_data_frame)

    def set_params(self, _experiment_config, _data_frame):
        self.experiment_config = _experiment_config
        self.data = _data_frame

    def set_likelihoods(self, likelihoods):
        self.likelihoods = likelihoods

    def get_all_parameters_z(self, runner):
        w_j = []
        for t in list(self.experiment_config.types.keys()):
            t -= 1

            # normalize just to make sure it is normalized
            runner.machines[2 + t].I_z = self.normalize_initial_z(runner.machines[2 + t].I_z)

            for state in runner.machines[2 + t].I:
                if runner.machines[2 + t].I[state] != LOG_EPS:
                    w_j.append(runner.machines[2 + t].I_z[state])

            for a in runner.machines[2 + t].T_z:
                # normalize just to make sure it is normalized
                runner.machines[2 + t].F_z, runner.machines[2 + t].T_z = self.normalize_a_state_new(runner.machines[2 + t].F_z, runner.machines[2 + t].T_z, a)

                for b in runner.machines[2 + t].T[a]:
                    for c in runner.machines[2 + t].T[a][b]:
                        w_j.append(runner.machines[2 + t].T_z[a][b][c])

            for state in runner.machines[2 + t].F:
                if runner.machines[2 + t].F[state] != LOG_EPS:
                    w_j.append(runner.machines[2 + t].F_z[state])

        return w_j

    def set_all_probabilities_z(self, runner, w_j_z, normalize=False):
        counter = 0
        temp = []
        for t in list(self.experiment_config.types.keys()):
            t -= 1
            for state in runner.machines[2 + t].I:
                if runner.machines[2 + t].I[state] != LOG_EPS:
                    temp.append(runner.machines[2 + t].I_z[state])
                    runner.machines[2 + t].I_z[state] = w_j_z[counter]
                    counter += 1

            for a in runner.machines[2 + t].T:
                for b in runner.machines[2 + t].T[a]:
                    for c in runner.machines[2 + t].T[a][b]:
                        temp.append(runner.machines[2 + t].T_z[a][b][c])
                        runner.machines[2 + t].T_z[a][b][c] = w_j_z[counter]
                        counter += 1

            for state in runner.machines[2 + t].F:
                if runner.machines[2 + t].F[state] != LOG_EPS:
                    temp.append(runner.machines[2 + t].F_z[state])
                    runner.machines[2 + t].F_z[state] = w_j_z[counter]
                    counter += 1

            if normalize:
                runner.machines[2 + t].F_z, runner.machines[2 + t].T_z = self.normalize_a_state_new(runner.machines[2 + t].F_z, runner.machines[2 + t].T_z, state)
                runner.machines[2 + t].F, runner.machines[2 + t].T = runner.machines[2 + t].F_z, runner.machines[2 + t].T_z

                runner.machines[2 + t].I_z = self.normalize_initial(runner.machines[2 + t].I_z)
                runner.machines[2 + t].I = runner.machines[2 + t].I_z

        return runner, temp

    ### NORMALIZATION METHODS ###
    def normalize_a_state(self, F, T, a):
        # find maximum log probability
        log_mx = LOG_EPS
        for b in T[a]:
            for c in T[a][b]:
                if T[a][b][c] > log_mx:
                    log_mx = T[a][b][c]
        # sum
        sm = 0
        for b in T[a]:
            for c in T[a][b]:
                sm += np.exp(T[a][b][c] - log_mx)

        if F[a] != LOG_EPS:
            if log_mx == LOG_EPS:
                sm += 1.
            else:
                sm += np.exp(F[a] - log_mx)

        # normalize
        for b in T[a]:
            for c in T[a][b]:
                T[a][b][c] = np.log(np.exp(T[a][b][c] - log_mx)/sm)
        if F[a] != LOG_EPS:
            if log_mx == LOG_EPS:
                F[a] = 0.
            else:
                F[a] = np.log(np.exp(F[a] - log_mx)/sm)

        return F, T

    def normalize_a_state_new(self, F, T, a):
        # find maximum log probability
        params = []
        for b in T[a]:
            for c in T[a][b]:
                params.append(T[a][b][c])

        if F[a] != LOG_EPS:
            params.append(F[a])

        log_mx = max(params)
        sm = sum([np.exp(param - log_mx) for param in params])

        # normalize
        for b in T[a]:
            for c in T[a][b]:
                T[a][b][c] = np.log(np.exp(T[a][b][c] - log_mx)/sm)
        if F[a] != LOG_EPS:
            if log_mx == LOG_EPS:
                F[a] = 0.
            else:
                F[a] = np.log(np.exp(F[a] - log_mx)/sm)

        return F, T

    def normalize_initial(self, I):
        # find maximum log probability
        log_mx = LOG_EPS
        for a in I:
            if I[a] != LOG_EPS and I[a]>log_mx:
                log_mx = I[a]
        # sum
        sm = 0
        for a in I:
            if I[a] != LOG_EPS:
                sm += np.exp(I[a] - log_mx)

        # normalize
        for a in I:
            if I[a] != LOG_EPS:
                I[a] = I[a] - log_mx - np.log(sm)

        return I

    def normalize_initial_z(self, I_z):
        # I = deepcopy(I)
        # find maximum log probability
        log_mx = LOG_EPS
        for a in I_z:
            if I_z[a] != LOG_EPS and I_z[a]>log_mx:
                log_mx = I_z[a]
        # sum
        sm = 0
        for a in I_z:
            if I_z[a] != LOG_EPS:
                sm += np.exp(I_z[a] - log_mx)

        # normalize
        for a in I_z:
            if I_z[a] != LOG_EPS:
                I_z[a] = I_z[a] - log_mx - np.log(sm)

        return I_z

    def normalize_final(self, F, T):
        for state in F:
            F, T = self.normalize_a_state_new(F, T, state)

        return F, T


from django.shortcuts import render, redirect
from main.models import Book, Customer, Address
from cart.models import Cart
from .models import Order, Rent
import datetime
from django.contrib import messages

# Create your views here.

def checkout(request):
    cart_objs = Cart.objects.filter(customer=request.user)
    items=[]
    for item in cart_objs:
        if not item.is_ordered:
            items.append(item)

    if len(items) == 0:
        return redirect('/cart/0')
    order = None
    for item in items:
        if item.order and not item.is_ordered:
            order = item.order
            break

    if order:
        pass
    else:
        order = Order.objects.create()

    for item in items:
        item.order = order
        item.save()

    order.customer = request.user
    order.total_cost = sum([item.total for item in items])
    order.save()
    addresses = Address.objects.filter(customer=request.user)
    items = Cart.objects.filter(customer=request.user, order=order)
    total = sum([item.total for item in items])

    return render(request, 'checkout.html', {'addresses': addresses, 'items': items, 'total': total, 'order_id': order.id})

def place(request, order_id):
    try:
        address_choice = request.GET['address']
        address_id = request.GET.get('c_address')
    except:
        messages.error(request, 'Жалғастыру үшін мекенжайды таңдаңыз. ',
                       extra_tags='address')
        return redirect('/checkout')

    if address_choice == '1':
        if address_id == None:
            messages.error(request, 'Жалғастыру үшін мекенжайды таңдаңыз.',
                        extra_tags='address')
            return redirect('/checkout')

    fname = request.GET['c_fname']
    lname = request.GET['c_lname']
    add_ln1 = request.GET['c_add_ln1']
    add_ln2 = request.GET['c_add_ln2']
    pin = request.GET['c_pin']
    city = request.GET['c_city']
    phone = request.GET['c_phone']

    if fname==None or lname==None or add_ln1==None or pin==None or phone==None or city==None:
        messages.error(
            request, 'Белгіленген (*) міндетті өрістерді толтыру қажет', extra_tags='field')
        return redirect('/checkout')

    order = Order.objects.get(id=order_id)

    if request.user.deposite < order.total_cost:
        messages.error(request, 'Тапсырысты рәсімдеу үшін сіздің шотыңызға тапсырыс сомасынан асатын сома енгізілуі керек.',
                        extra_tags='deposite')
        return redirect('/checkout')

    if address_choice == '1':
        address = Address.objects.get(id=address_id)

    else:
        address = Address.objects.create(first_name=fname, last_name=lname, add_ln1=add_ln1, add_ln2=add_ln2, city=city,
                                         pincode=pin, phone=phone, customer=request.user)

    order.address = address
    order.is_ordered = True
    order.date_ordered = datetime.datetime.now()
    order.save()
    items = Cart.objects.filter(customer=request.user)
    for item in items:
        item.is_ordered = True
        item.save()
    rent = Rent.objects.create(order=order)
    rent.save()
    return render(request, 'thankyou.html')


from django.views.generic import DetailView, RedirectView
from django.views.generic.detail import SingleObjectMixin
from django.contrib.auth.mixins import LoginRequiredMixin

from .models import Battle, Challenger

import os
import random

class RandomQuestion():
    def question(self):
        path = "python_questions"
        append_string = ''
        list_of_strings = []
        names = os.listdir(path)
        # removes hidden files, just in case.
        names = [f for f in names if not f.startswith('.')]
        list_of_paths = []
        for root, dirs, files in os.walk(path):
            for f in files:
                x = os.path.join(root, f)
                list_of_paths.append(x)

        question = random.choice(list_of_paths)
        open_question = open(question, 'r')
        display_question = open_question.read()
        display_question = display_question[3:-4]

        for i in display_question:
            if i != '\n':
                append_string += i
            if i == '/':
                list_of_strings.append(append_string[0:-1])
                append_string = ""
        return list_of_strings

    def name(self):
        path = "python_questions"
        names = os.listdir(path)
        # removes hidden files, just in case.
        names = [f for f in names if not f.startswith('.')]
        name = random.choice(names)
        return name

my_class = RandomQuestion()

class BattleView(DetailView):
    model = Battle
    template_name = 'battle/battle.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['battle_id'] = self.get_object().pk
        return context

class PreBattleView(LoginRequiredMixin, SingleObjectMixin, RedirectView):
    model = Battle
    pattern_name = 'battle'

    def get(self, request, *args, **kwargs):
        queryset = self.get_queryset()
        if request.user.profile.in_battle():
            self.current_pk = request.user.profile.current_battle().pk
        elif queryset.filter(state=Battle.WAITING).count() > 0:
            battle = queryset.filter(state=Battle.WAITING).order_by('-pk')[0]
            if battle.challengers.count() == 0:
                battle.challengers.add(request.user)
                battle.save()
                self.current_pk = battle.pk
            else:
                color = Challenger.BLUE
                if battle.challenger_set.get().color == Challenger.BLUE:
                    color = Challenger.RED
                battle.challengers.add(request.user, through_defaults={'color': color})
                battle.state = Battle.READY
                battle.save()
                self.current_pk = battle.pk
        else:
            battle = Battle.objects.create()
            battle.challengers.add(request.user)
            battle.save()
            self.current_pk = battle.pk
        if request.user.battles.exclude(state=Battle.ARCHIVED).count() > 2:
            print('DEBUG LN 89 battle/views.py')
            request.user.challenges.exclude(
                battle__state=Battle.ARCHIVED, battle__pk=self.current_pk
            ).delete()
        return super().get(request, *args, **kwargs)

    def get_redirect_url(self, *args, **kwargs):
        return super().get_redirect_url(self.current_pk, *args, **kwargs)


import pickle
from random import sample
from sklearn.externals import joblib
from sklearn.linear_model import LogisticRegression
from numpy import set_printoptions , inf

def accuarcy(a,b):
    i = 0
    count = 0
    for i in range(len(a)):
        if a[i] == b[i]:
            count = count + 1
    return count/len(a)
set_printoptions(threshold=inf)
data = open("../notMNIST.pickle","rb")

values = pickle.load(data)

test_dataset = values['test_dataset']

nsamples , nx , ny = test_dataset.shape

test_dataset = test_dataset.reshape((nsamples,nx*ny))

#valid_dataset = sample(valid_dataset,10)
model = joblib.load('initial_model_saga_10000.pkl')

result = model.predict(test_dataset)

print(accuarcy(result,values['test_labels']))
#print(result)
#print(values['valid_labels'])
#print(type(model))


import datetime
import hashlib

import app
import app.model
import settings

def delete_token(ip_address, stoken):
    token = app.model.Form()
    token.token = stoken
    token.created_by_ip = ip_address

    try:
        token.delete()
        result = True
    except:
        result = False

    return result

def has_valid_token(ip_address, token):
    tokens = app.model.Form.all()
    tokens.filter("token =", token)
    tokens.filter("created_by_ip =", ip_address)
    tokens.filter("expired_at >", datetime.datetime.now())
    return tokens.get() != None

def put_form_token(ip_address, token=""):
    if token == "":
        token = make_token()

    ftoken = app.model.Form()
    ftoken.token = token
    ftoken.created_at = datetime.datetime.now()
    ftoken.created_by_ip = ip_address
    ftoken.expired_at = datetime.datetime.now() + settings.PASTE_FORM_EXPIRATION_DELTA
    key = ftoken.put()
    if key != "":
        return token
    else:
        return ""

def make_token():
    hash = hashlib.sha256()
    hash.update(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:BONOBO"))
    return hash.hexdigest()


#pip list: 파이썬 모듈들 표시
#'pip install (streamlit)파이썬 모듈 이름'으로 설치
#streamlit --version
#conda activate base: base로
#streamlit: 웹 앱 만드는데 최적화, 웹브라우저에 출력

import streamlit as st
st.text("streamlit ver. : " + st.__version__) #text함수 이용해 버전 출력
#'streamlit run st_01_start.py' 출력 실행

# ## Text
# ## Title
st.title('Streamlit Tutorial: start by pdm00, 이상훈')
# ## Header/Subheader
st.header('This is header')
st.subheader('This is subheader')
# ## Text
st.text("Hello Streamlit! 이 글은 튜토리얼 입니다.")

# ## Markdown
st.header('Markdown')
st.markdown('# header-1')
st.markdown('## header-2')
st.markdown('#### header-4')
st.markdown('###### header-6')

# # list
st.header('markdown: List')
st.markdown('- list1')
# st.markdown('list1')
st.markdown('- list2')
st.markdown('- list3\n'
            '   * inner list1\n'
            '   * inner list2\n'
            '       - inner_inner list\n')

# ## Message
st.info("End of first note: **text, markdown, latex**!") #info(문자): 색 있는 타이틀 바 - 문자 강조

## streamlit messages <안내 추가>
st.success("Successful") #초록
st.info("Information!") #밝은 하늘
st.warning("This is a warning") #노랑
st.error("This is an error!") #빨강
st.exception("NameError('Error name is not defined')") #빨강 - 에러의 일종

## Code block
st.header('Code block')
code1 = '''def hello():
    print("Hello, Streamlit!")''' #''' ''', """ """: 두 줄 이상의 문자열// 콜론 후 들여쓰기

st.code(code1, language='python') #코드가 파이썬 포맷으로, 회색 배경

code2 = '''
def add(a,b):
    return a+b
'''

st.code(code2, language='python')

# ## Latex, raw
st.header('latex') #수식
st.latex(r"\alpha_n, \beta, \gamma, \cdots, \omega") #r: 특수코드 없는 문자열, _n:아래첨자 n, \:그리스어
st.latex(r"Y = \alpha + \beta X_i") #1차원 회귀식
# ## Latex-inline
st.markdown(r"회귀분석에서 오차는 다음과 같습니다 $e_i = y_i - \hat{y}_i$") #라텍스 inline 수식 - $안에 수식 넣기
# ## Mean squared error
st.subheader("Mean squared error")
st.markdown(r"#### $MSE = \frac{1}{N-1} \sum_{i=1}^{N-1} (y_i - \hat{y}_i)^2$")


from django.shortcuts import render
from django.http import HttpResponse
from django.conf import settings
from django.conf.urls.static import static
from django.contrib.staticfiles import finders
from django.template import Template, Context
import os
import yaml
from pathlib import Path
import mistune

# Create your views here.
def index(request):
    ctxt = {'projects': [
        {'image': 'images/image5.png',
         'name': 'Prediction of Steering Angles',
         'description': 'Using Keras to predict steering angles from images of the road',
         'link': 'https://google.com'},
        {'image': 'images/melee_thumb.jpg',
         'name': 'Super Smash Bros. Melee AI',
         'description': 'Creation of a Machine Learning Melee AI Using Keras',
         'link': 'https://google.com'},
        {'image': 'images/OW_thumb.jpg',
         'name': 'Overwatch Fight Prediction',
         'description': 'Final part of a Data Science project to predict the winners of fights in Overwatch',
         'link': 'https://google.com'},
        {'image': 'images/vlight_thumb.png',
         'name': 'Volumetric Lighting',
         'description': 'OpenGL implementation of Volumetric Lighting',
         'link': 'https://google.com'},
    ]}
    project_files_dir = finders.find('portfolio/projects/')
    project_files = os.listdir(project_files_dir)
    ctxt = {'projects': []}
    for project in project_files:
        project_path = Path(project_files_dir).joinpath(project)
        details, content = process_project_page(project_path)
        details['link'] = project.split('.')[0]
        ctxt['projects'].append(details)

    return render(request, 'portfolio/index.html', ctxt)

def process_project_page(project_page_path):
    with open(project_page_path) as pfile:
        project_details = ''
        for line in pfile:
            if line == '---\n':
                break
            project_details += line
        project_details = yaml.load(project_details, Loader=yaml.FullLoader)
        project_content = pfile.read()
    return project_details, project_content

def project_page(request, project_name):
    project_file = finders.find(f'portfolio/projects/{project_name}.md')
    details, content = process_project_page(project_file)
    content = Template(content).render(Context({}))
    content = mistune.html(content)
        #project_details = yaml.load(pfile, Loader=yaml.FullLoader)

    ctxt = {
        'name': project_name,
        'content': content,
    }
    ctxt.update(details)

    #ctxt = project_details
    return render(request, 'portfolio/project.html', ctxt)

def resume(request):
    return render(request, 'portfolio/resume.html')


'''
File: Audiolyze.py
Author: Tristan van Vaalen

Audio stream analyzer
'''

import config
import microphone
from audiolazy import z
# import audiolazy

class Audiolyze:

    def __init__(self, filt):
        self.config = config.Config()
        self.microphone = microphone.Microphone(filt, **self.config.args)

if __name__ == '__main__':
    # filt = (0.2 * (1 - z ** 5)) / (z ** 4)(1 - z)
    # filt = 0.2 * (1 + z ** -1 + z ** -2 + z ** -3 + z ** -4)
    # filt = 0.2 * (1 - z**5) / (z**2 * (1-z))
    # filt = 0.03824 * (z**2 - 1) / (z**2 + 0.9235)
    # filt = (-z + 3)/(z - 9.5)
    # filt = (0.18 * z) / ((z - 1) * (z - 0.82))
    # filt = audiolazy.comb.tau(delay=30*2, tau=40*4)
    filt = 1 / ((2 * (z - 1)) / (0.1 * (z + 1)) + 1)
    # filt = 1 - z ** -1
    audiolyze = Audiolyze(filt)