import numpy as np

a = np.arange(12)**2                       # the first 12 square numbers
i = np.array( [ 1,1,3,8,5 ] )              # an array of indices
print(a[i])                                       # the elements of a at the positions i
# array([ 1,  1,  9, 64, 25])

j = np.array( [ [ 3, 4], [ 9, 7 ] ] )      # a bidimensional array of indices
print(a[j])                                       # the same shape as j
# array([[ 9, 16],
#        [81, 49]])

# When the indexed array a is multidimensional, a single array of indices refers to the first dimension of a.
# The following example shows this behavior by converting an image of labels into a color image using a palette.
palette = np.array( [ [0,0,0],                # black
                      [255,0,0],              # red
                      [0,255,0],              # green
                      [0,0,255],              # blue
                      [255,255,255] ] )       # white
image = np.array( [ [ 0, 1, 2, 0 ],           # each value corresponds to a color in the palette
                    [ 0, 3, 4, 0 ]  ] )
print(palette[image])                            # the (2,4,3) color image
# array([[[  0,   0,   0],
#         [255,   0,   0],
#         [  0, 255,   0],
#         [  0,   0,   0]],
#        [[  0,   0,   0],
#         [  0,   0, 255],
#         [255, 255, 255],
#         [  0,   0,   0]]])

# We can also give indexes for more than one dimension. The arrays of indices for each dimension must have the same shape.
a = np.arange(12).reshape(3,4)
print(a)
# array([[ 0,  1,  2,  3],
#        [ 4,  5,  6,  7],
#        [ 8,  9, 10, 11]])
i = np.array( [ [0,1],                        # indices for the first dim of a
                [1,2] ] )
j = np.array( [ [2,1],                        # indices for the second dim
                [3,3] ] )
print(a[i,j])                                    # i and j must have equal shape
# array([[ 2,  5],
#        [ 7, 11]])
print(a[i,2])
# array([[ 2,  6],
#        [ 6, 10]])
print(a[:,j])                                     # i.e., a[ : , j]
# array([[[ 2,  1],
#         [ 3,  3]],
#        [[ 6,  5],
#         [ 7,  7]],
#        [[10,  9],
#         [11, 11]]])

# Naturally, we can put i and j in a sequence (say a list) and then do the indexing with the list.
l = [i,j]
print(a[l])                                       # equivalent to a[i,j]
# array([[ 2,  5],
#        [ 7, 11]])

# However, we can not do this by putting i and j into an array, because this array will be interpreted as indexing the first dimension of a.
s = np.array( [i,j] )
# print(a[s])                                       # not what we want
# Traceback (most recent call last):
#   File "<stdin>", line 1, in ?
# IndexError: index (3) out of range (0<=index<=2) in dimension 0
print(a[tuple(s)])                              # same as a[i,j]
# array([[ 2,  5],
#        [ 7, 11]])

# Another common use of indexing with arrays is the search of the maximum value of time-dependent series:
time = np.linspace(20, 145, 5)                 # time scale
data = np.sin(np.arange(20)).reshape(5,4)      # 4 time-dependent series
print(time)
# array([  20.  ,   51.25,   82.5 ,  113.75,  145.  ])
print(data)
# array([[ 0.        ,  0.84147098,  0.90929743,  0.14112001],
#        [-0.7568025 , -0.95892427, -0.2794155 ,  0.6569866 ],
#        [ 0.98935825,  0.41211849, -0.54402111, -0.99999021],
#        [-0.53657292,  0.42016704,  0.99060736,  0.65028784],
#        [-0.28790332, -0.96139749, -0.75098725,  0.14987721]])

ind = data.argmax(axis=0)                   # index of the maxima for each series
print(ind)
# array([2, 0, 3, 1])
time_max = time[ ind]                       # times corresponding to the maxima
data_max = data[ind, xrange(data.shape[1])] # => data[ind[0],0], data[ind[1],1]...
print(time_max)
# array([  82.5 ,   20.  ,  113.75,   51.25])
print(data_max)
# array([ 0.98935825,  0.84147098,  0.99060736,  0.6569866 ])
print(np.all(data_max == data.max(axis=0)))
# True

# You can also use indexing with arrays as a target to assign to:
a = np.arange(5)
print(a)
# array([0, 1, 2, 3, 4])
a[[1,3,4]] = 0
print(a)
# array([0, 0, 2, 0, 0])

# However, when the list of indices contains repetitions, the assignment is done several times, leaving behind the last value:
a = np.arange(5)
a[[0,0,2]]=[1,2,3]
print(a)
# array([2, 1, 3, 3, 4])

# This is reasonable enough, but watch out if you want to use Python's += construct, as it may not do what you expect:
a = np.arange(5)
a[[0,0,2]]+=1
print(a)
# array([1, 1, 3, 3, 4])

# Even though 0 occurs twice in the list of indices, the 0th element is only incremented once.
# This is because Python requires "a+=1" to be equivalent to "a = a + 1".

numpy/tutorial/fancy indexing and index tricks/index_with_array_of_indices.py


#!/usr/bin/env python

def cookie(cookies, case):
	c = cookies[0]
	f = cookies[1]
	x = cookies[2]

	cookiesPerSecond = 2.0

	totalTimeForCookiesFarm = 0.0
	# Start with worst case?
	lastTimeTotal = x / cookiesPerSecond

	while (1):
		# Increase last total time to buy farm
		totalTimeForCookiesFarm += c / cookiesPerSecond
		# Boost Speed after farm is bought
		cookiesPerSecond = cookiesPerSecond + f

		# See if we're getting slower or faster in producing cookies
		currentTotalTime = (x / cookiesPerSecond) + totalTimeForCookiesFarm
		if currentTotalTime < lastTimeTotal:
			lastTimeTotal = currentTotalTime
		else:
			break

	print("Case #%d: %.7f" % (case, lastTimeTotal))

numCases = int(input())

for case in range(0, numCases):
	cookies = [float(i) for i in raw_input().strip().split()]
	# print cookies
	cookie(cookies, case+1)


""" Demographics - Fraction of Patients deemed severe by WHO Ordinal Scale (score > 4)"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.colors import LinearSegmentedColormap
import matplotlib
matplotlib.rc('font', family='sans-serif')
plt.style.use('ggplot')
def make_autopct(values):
    def my_autopct(pct):
        total = sum(values)
        val = int(round(pct*total/100.0))
        return '{p:.2f}%  ({v:d})'.format(p=pct,v=val) if pct > 0 else ''
    return my_autopct

#get labels
df = pd.read_csv('../../Data/ImmuneCODE/ImmuneCODE-Repertoire-Tags-002.2.tsv',sep='\t')
df['who_ordinal_scale_bin'] = None
df['who_ordinal_scale_bin'][df['who_ordinal_scale'] <=4] = 'mild'
df['who_ordinal_scale_bin'][df['who_ordinal_scale'] > 4] = 'severe'
df['who_ordinal_scale'] = df['who_ordinal_scale_bin']
df['Age'] = df['Age'].str.split(' ',expand=True)[0]
df['Age'] = df['Age'].astype(float)

ds = ['COVID-19-ISB']
if ds is not None:
    df = df[df['Dataset'].isin(ds)]

label_sel = 'who_ordinal_scale'
c_list = []
data_list = []
for c in np.unique(df['Dataset']):
    df_c = df[df['Dataset'] == c]
    df_c.dropna(subset=[label_sel],inplace=True)
    data_list.append(df_c[label_sel])
    c_list.append(len(df_c)*[c])

df_data = pd.DataFrame()
df_data['c'] = np.hstack(c_list)
df_data[label_sel] = np.hstack(data_list)
df_data.to_csv('data/who.csv',index=False)

#categorical label
df_data = pd.DataFrame(df_data.groupby(['c',label_sel]).size())
df_data.reset_index(inplace=True)
df_data = pd.pivot(df_data,'c',label_sel,values=0)
df_data.fillna(value=0,inplace=True)
sum_rows = np.sum(df_data,1)

labels = np.array(list(df_data.columns))
d = df_data.index[0]

fig1, ax = plt.subplots(1,1,figsize=(5,5))
values = np.array(df_data.loc[d])
patches,text,_ = ax.pie(values, labels=labels, autopct=make_autopct(values), startangle=90,textprops={'fontsize':12})
ax.axis('equal')
ax.set_title(d)
for ii,t in enumerate(text,0):
    if values[ii] == 0:
        t.set_text('')
plt.tight_layout()
plt.savefig('figures/who_ordinal_scale.eps')


import numpy as np
import matplotlib.pyplot as plt
import cv2
from util import image_process as ip

def plot_histogram(warped, image_name='', out_dir=None):
    '''
    Returns histogram peaks along horizontal axis of image, giving one each for left and right half.
    '''
    binary_warped =  warped.copy()
    # Take a histogram of the bottom half of the image
    histogram = np.sum(warped[int(binary_warped.shape[0]/2):,:], axis=0)

    # Peak in the first half indicates the likely position of the left lane
    half_width = np.int(histogram.shape[0]/2)
    leftx_base = np.argmax(histogram[:half_width])

    # Peak in the second half indicates the likely position of the right lane
    rightx_base = np.argmax(histogram[half_width:]) + half_width

    if out_dir:
        plt.figure(figsize=(10,12))
        plt.plot(histogram)
        out_title = '{}_histogram'.format(image_name)
        plt.savefig('{}/{}.jpg'.format(out_dir, out_title))
    return(leftx_base, rightx_base)

def polyfit_sliding_window(warped, image_name='', out_dir=None):
    '''
    Returns pixel points within the search area(left and right half of image) performing a sliding window search on a warped image.
    Also returns a second order polynomial that fits the pixel points identified in the two regions.
    '''
    if out_dir:
        plt.figure(figsize=(10,12))
    binary_warped = warped.copy()
    # Create an output image to draw on and  visualize the result
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255

    leftx_base, rightx_base = plot_histogram(binary_warped)

    # Choose the number of sliding windows
    nwindows = 9
    # Set height of windows
    window_height = np.int(binary_warped.shape[0]/nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated for each window
    leftx_current = leftx_base
    rightx_current = rightx_base
    # Set the width of the windows +/- margin
    margin = 100
    # Set minimum number of pixels found to recenter window
    minpix = 50
    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = binary_warped.shape[0] - (window+1)*window_height
        win_y_high = binary_warped.shape[0] - window*window_height
        win_xleft_low = leftx_current - margin
        win_xleft_high = leftx_current + margin
        win_xright_low = rightx_current - margin
        win_xright_high = rightx_current + margin

          # Draw the windows on the visualization image
        cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),
            (0,255,0), 2)
        cv2.rectangle(out_img,(win_xright_low,win_y_low),(win_xright_high,win_y_high),
                (0,255,0), 2)

        # Identify the nonzero pixels in x and y within the window
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
        (nonzerox >= win_xleft_low) &  (nonzerox < win_xleft_high)).nonzero()[0]
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
        (nonzerox >= win_xright_low) &  (nonzerox < win_xright_high)).nonzero()[0]
        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)
        # If you found > minpix pixels, recenter next window on their mean position
        if len(good_left_inds) > minpix:
            leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
        if len(good_right_inds) > minpix:
            rightx_current = np.int(np.mean(nonzerox[good_right_inds]))

    # Concatenate the arrays of indices
    left_lane_inds = np.concatenate(left_lane_inds)
    right_lane_inds = np.concatenate(right_lane_inds)

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds]
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]

    # Fit a second order polynomial to each
    left_fit_p, right_fit_p = (None, None)
    # Fit a second order polynomial to each
    if len(leftx) != 0:
        left_fit_p = np.polyfit(lefty, leftx, 2)
    if len(rightx) != 0:
        right_fit_p = np.polyfit(righty, rightx, 2)

    if out_dir:
        # Generate x and y values for plotting
        ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
        left_fitx = left_fit_p[0]*ploty**2 + left_fit_p[1]*ploty + left_fit_p[2]
        right_fitx = right_fit_p[0]*ploty**2 + right_fit_p[1]*ploty + right_fit_p[2]

        out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
        out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]
        plt.imshow(out_img)
        plt.plot(left_fitx, ploty, color='yellow')
        plt.plot(right_fitx, ploty, color='yellow')
        plt.xlim(0, 1280)
        plt.ylim(720, 0)
        out_title = '{}_slidingwindow'.format(image_name)
        plt.savefig('{}/{}.jpg'.format(out_dir, out_title))

    return (left_fit_p, right_fit_p, left_lane_inds, right_lane_inds)

def polyfit_using_previous_fit(warped, left_fit, right_fit, image_name='', out_dir=None):
    '''
    Returns pixel points within the search area(left and right half of image) performing a pixel on a warped image,
    around region spsecified by the fitting polynomials from previous fit.
    Also returns a second order polynomial that fits the pixel points identified in the two regions.
    '''
    if out_dir:
        plt.figure(figsize=(10,12))

    binary_warped = warped.copy()
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    margin = 100

    left_lane_inds = ((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy +
    left_fit[2] - margin)) & (nonzerox < (left_fit[0]*(nonzeroy**2) +
    left_fit[1]*nonzeroy + left_fit[2] + margin)))

    right_lane_inds = ((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy +
    right_fit[2] - margin)) & (nonzerox < (right_fit[0]*(nonzeroy**2) +
    right_fit[1]*nonzeroy + right_fit[2] + margin)))

    # Again, extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds]
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]
    # Fit a second order polynomial to each
    left_fit_p, right_fit_p = (None, None)
    if len(leftx) != 0:
        left_fit_p = np.polyfit(lefty, leftx, 2)
    if len(rightx) != 0:
        right_fit_p = np.polyfit(righty, rightx, 2)

    if out_dir:
        # Generate x and y values for plotting
        ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
        left_fitx = left_fit_p[0]*ploty**2 + left_fit_p[1]*ploty + left_fit_p[2]
        right_fitx = right_fit_p[0]*ploty**2 + right_fit_p[1]*ploty + right_fit_p[2]

        # Create an image to draw on and an image to show the selection window
        out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255
        window_img = np.zeros_like(out_img)
        # Color in left and right line pixels
        out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
        out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]

        # Generate a polygon to illustrate the search window area
        # And recast the x and y points into usable format for cv2.fillPoly()
        left_line_window1 = np.array([np.transpose(np.vstack([left_fitx-margin, ploty]))])
        left_line_window2 = np.array([np.flipud(np.transpose(np.vstack([left_fitx+margin,
                                      ploty])))])
        left_line_pts = np.hstack((left_line_window1, left_line_window2))
        right_line_window1 = np.array([np.transpose(np.vstack([right_fitx-margin, ploty]))])
        right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_fitx+margin,
                                      ploty])))])
        right_line_pts = np.hstack((right_line_window1, right_line_window2))

        # Draw the lane onto the warped blank image
        cv2.fillPoly(window_img, np.int_([left_line_pts]), (0,255, 0))
        cv2.fillPoly(window_img, np.int_([right_line_pts]), (0,255, 0))
        result = cv2.addWeighted(out_img, 1, window_img, 0.3, 0)
        plt.imshow(result)
        plt.plot(left_fitx, ploty, color='yellow')
        plt.plot(right_fitx, ploty, color='yellow')
        plt.xlim(0, 1280)
        plt.ylim(720, 0)
        out_title = '{}_previousfit'.format(image_name)
        plt.savefig('{}/{}.jpg'.format(out_dir, out_title))

    return (left_fit_p, right_fit_p, left_lane_inds, right_lane_inds)

def measure_radius_of_curvature(bin_img, left_fit_p, right_fit_p, left_lane_inds, right_lane_inds):
    '''
    Returns radius of curvature of lanes in the input binary image
    '''
    # Define conversions in x and y from pixels space to meters
    ym_per_pix = 30/720 # meters per pixel in y dimension
    xm_per_pix = 3.7/700 # meters per pixel in x dimension

    h = bin_img.shape[0]
    ploty = np.linspace(0, h-1, h)
    y_eval = np.max(ploty)

     # Identify the x and y positions of all nonzero pixels in the image
    nonzero = bin_img.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Again, extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds]
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]

    # Fit new polynomials to x,y in world space
    if len(leftx) != 0 and len(rightx) != 0:
        left_fit_cr = np.polyfit(lefty*ym_per_pix, leftx*xm_per_pix, 2)
        right_fit_cr = np.polyfit(righty*ym_per_pix, rightx*xm_per_pix, 2)
        # Calculate the new radii of curvature
        left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
        right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
        # Now our radius of curvature is in meters
        curve_rad = (left_curverad + right_curverad)/2
    else:
        curve_rad = 0
    return curve_rad

def measure_offset_from_center(bin_img, left_fit_p, right_fit_p):
    '''
    Returns offset distance from center for lanes in the input binary image
    '''
    if left_fit_p is not None and right_fit_p is not None:
        # compute the offset from the center
        h = bin_img.shape[0]
        l_fit_x = left_fit_p[0]*h**2 + left_fit_p[1]*h + left_fit_p[2]
        r_fit_x = right_fit_p[0]*h**2 + right_fit_p[1]*h + right_fit_p[2]
        lane_center = (r_fit_x + l_fit_x) /2
        xm_per_pix = 3.7/700 # meters per pixel in x dimension
        center_offset_pixels = abs(bin_img.shape[1]/2 - lane_center)
        center_offset = xm_per_pix*center_offset_pixels
    else:
        center_offset = 0
    return center_offset

def shade_lane_area(img, binary_img, l_fit, r_fit, Minv, image_name='', out_dir=None):
    '''
    Shade the lane area in the original lane image.
    '''
    if l_fit is None or r_fit is None:
        return img
    # Create an image to draw the lines on
    warp_zero = np.zeros_like(binary_img).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))

    h,w = binary_img.shape
    ploty = np.linspace(0, h-1, num=h)# to cover same y-range as image
    left_fitx = l_fit[0]*ploty**2 + l_fit[1]*ploty + l_fit[2]
    right_fitx = r_fit[0]*ploty**2 + r_fit[1]*ploty + r_fit[2]

    # Recast the x and y points into usable format for cv2.fillPoly()
    pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
    pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
    pts = np.hstack((pts_left, pts_right))

    # Draw the lane onto the warped blank image
    cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))
    cv2.polylines(color_warp, np.int32([pts_left]), isClosed=False, color=(255,0,255), thickness=15)
    cv2.polylines(color_warp, np.int32([pts_right]), isClosed=False, color=(0,255,255), thickness=15)

    # Warp the blank back to original image space using inverse perspective matrix (Minv)
    newwarp = cv2.warpPerspective(color_warp, Minv, (w, h))
    # Combine the result with the original image
    result = cv2.addWeighted(img, 1, newwarp, 0.3, 0)
    if out_dir:
        plt.figure(figsize=(10,12))
        out_title = '{}_shadedlane'.format(image_name)
        plt.savefig('{}/{}.jpg'.format(out_dir, out_title))
    return result

def add_distance_text_to_image(img, curv_rad, center_offset, image_name='', out_dir=None):
    '''
    Adds distance measurements as text on top of a lane shaded image
    '''
    height = img.shape[0]
    font = cv2.FONT_HERSHEY_PLAIN
    text = 'Radius of Curvature: ' + '{:04.2f}'.format(curv_rad) + 'm'
    cv2.putText(img, text, (40,70), font, 3, (200,255,155), 2, cv2.LINE_AA)
    direction = ''
    if center_offset > 0:
        direction = 'right'
    elif center_offset < 0:
        direction = 'left'
    abs_center_offset = abs(center_offset)
    text = '{:04.3f}'.format(abs_center_offset) + 'm ' + direction + ' of center'
    cv2.putText(img, text, (40,120), font, 3, (200,255,155), 2, cv2.LINE_AA)
    if out_dir:
        out_title = '{}_shadedlane_curvature'.format(image_name)
        plt.savefig('{}/{}.jpg'.format(out_dir, out_title))
    return img


from config import CHANNEL
import re
from urllib.request import urlopen

def cmd(e, c, msg):
        if not msg:
            return
        msg = msg.replace(' ', '%20')
        html = urlopen('http://www.sloganizer.net/en/outbound.php?slogan='
                       + msg, timeout=1).read().decode()
        match = re.search('>(.*)<', html)
        c.privmsg(CHANNEL, match.group(1))


import unittest

import numpy
import six

import chainer
from chainer.backends import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr

@testing.parameterize(*(
    testing.product({
        'shape': [(3, 2, 4)],
        'axis': [None, 0, 1, 2, -1, (0, 1), (1, -1)],
        'dtype': [numpy.float16, numpy.float32, numpy.float64],
        'use_weights': [True, False],
        'keepdims': [True, False],
    }) +
    testing.product({
        'shape': [()],
        'axis': [None],
        'dtype': [numpy.float16, numpy.float32, numpy.float64],
        'use_weights': [True, False],
        'keepdims': [True, False],
    })))
class TestAverage(unittest.TestCase):

    def setUp(self):
        ndim = len(self.shape)
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        if self.axis is None:
            w_shape = self.shape
        elif isinstance(self.axis, int):
            axis = self.axis
            if axis < 0:
                axis += ndim
            w_shape = self.shape[axis],
        else:
            w_shape = tuple(self.shape[a] for a in self.axis)

        g_shape = self.x.sum(axis=self.axis, keepdims=self.keepdims).shape
        self.gy = numpy.random.uniform(-1, 1, g_shape).astype(self.dtype)

        # Sample weights. Weights should not sum to 0.
        while True:
            self.w = numpy.random.uniform(-2, 2, w_shape).astype(self.dtype)
            if self.w.sum() > 1e-2:
                break

        if self.dtype == numpy.float16:
            self.check_backward_options = {'atol': 1e-2, 'rtol': 1e-1}
        else:
            self.check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}

    def check_forward(self, x_data, axis, weights):
        if self.use_weights and isinstance(self.axis, tuple):
            # This condition is not supported
            return

        x = chainer.Variable(x_data)
        if self.use_weights:
            w = chainer.Variable(weights)
            w_data = self.w
        else:
            w = None
            w_data = None
        y = functions.average(x, axis=axis, weights=w, keepdims=self.keepdims)
        self.assertEqual(y.data.dtype, self.dtype)
        y_expect = numpy.average(
            self.x, axis=axis, weights=w_data)
        if self.keepdims:
            # numpy.average does not support keepdims
            if axis is None:
                axis = list(six.moves.range(x_data.ndim))
            elif isinstance(axis, int):
                axis = axis,
            shape = list(x_data.shape)
            for i in six.moves.range(len(shape)):
                if i in axis or i - len(shape) in axis:
                    shape[i] = 1
            y_expect = y_expect.reshape(shape)

        if self.dtype == numpy.float16:
            options = {'atol': 5e-3, 'rtol': 5e-3}
        else:
            options = {}

        self.assertEqual(y_expect.shape, y.shape)
        testing.assert_allclose(y_expect, y.data, **options)

    def test_forward_cpu(self):
        self.check_forward(self.x, self.axis, self.w)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(
            cuda.to_gpu(self.x), self.axis, cuda.to_gpu(self.w))

    def check_backward(self, x_data, y_grad, axis, w_data):
        if self.use_weights and isinstance(self.axis, tuple):
            # This condition is not supported
            return

        if self.use_weights:
            def f(x, w):
                return functions.average(
                    x, axis=axis, weights=w, keepdims=self.keepdims)
            args = (x_data, w_data)
        else:
            def f(x):
                return functions.average(x, axis=axis, keepdims=self.keepdims)
            args = x_data

        gradient_check.check_backward(
            f, args, y_grad, dtype=numpy.float64,
            **self.check_backward_options)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy, self.axis, self.w)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.gy), self.axis,
            cuda.to_gpu(self.w))

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestAverageDuplicateValueInAxis(unittest.TestCase):

    def test_duplicate_value(self):
        x = numpy.random.uniform(-1, 1, 24).reshape(2, 3, 4).astype(self.dtype)
        with self.assertRaises(ValueError):
            functions.average(x, axis=(0, 0))

    def test_duplicate_value_negative(self):
        x = numpy.random.uniform(-1, 1, 24).reshape(2, 3, 4).astype(self.dtype)
        with self.assertRaises(ValueError):
            functions.average(x, axis=(1, -2))

    def test_weights_and_axis(self):
        x = numpy.random.uniform(-1, 1, 24).reshape(2, 3, 4).astype(self.dtype)
        w = numpy.random.uniform(-1, 1, 6).reshape(2, 3).astype(self.dtype)
        with self.assertRaises(ValueError):
            functions.average(x, axis=(0, 1), weights=w)

testing.run_module(__name__, __file__)


import os
import sys
import shutil
from subprocess import call
from annogesiclib.helper import Helper
from annogesiclib.multiparser import Multiparser
from annogesiclib.converter import Converter
from annogesiclib.merge_manual import merge_manual_predict_tss
from annogesiclib.stat_TSSpredator import stat_tsspredator
from annogesiclib.plot_TSS_venn import plot_venn
from annogesiclib.validate_gene import validate_gff
from annogesiclib.stat_TA_comparison import stat_ta_tss
from annogesiclib.check_orphan import check_orphan
from annogesiclib.filter_TSS_pro import filter_tss_pro
from annogesiclib.filter_low_expression import filter_low_expression

class TSSpredator(object):

    def __init__(self, args_tss):
        self.multiparser = Multiparser()
        self.helper = Helper()
        self.converter = Converter()
        self.master = os.path.join(args_tss.out_folder, "MasterTables")
        self.tmps = {"tss": "tmp_TSS", "ta_tss": "tmp_ta_tss", "tss_ta":
                     "tmp_tss", "tmp": "tmp"}
        if args_tss.ta_files is not None:
            self.tmps["ta"] = os.path.join(args_tss.ta_files, "tmp")
        else:
            self.tmps["ta"] = None
        self.gff_path = os.path.join(args_tss.gffs, "tmp")
        self.wig_path = os.path.join(args_tss.wig_folder, "tmp")
        self.fasta_path = os.path.join(args_tss.fastas, "tmp")
        self.stat_outfolder = os.path.join(args_tss.out_folder, "statistics")
        self.gff_outfolder = os.path.join(args_tss.out_folder, "gffs")

    def _assign_dict(self, lib_datas):
        return {"wig": lib_datas[0],
                "tex": lib_datas[1],
                "condition": int(lib_datas[2]),
                "replicate": lib_datas[3],
                "strand": lib_datas[4]}

    def _print_lib(self, lib_num, lib_list, out, wig_folder, prefix):
        for num_id in range(1, lib_num+1):
            cond_list = []
            for lib in lib_list:
                if num_id == lib["condition"]:
                    cond_list.append(lib)
            cond_sort_list = sorted(cond_list, key=lambda k: k['replicate'])
            for cond in cond_sort_list:
                out.write("{0}_{1}{2} = {3}\n".format(
                          prefix, cond["condition"], cond["replicate"],
                          os.path.join(wig_folder, cond["wig"])))

    def _start_to_run(self, tsspredator_path, config_file, out_path, prefix):
        print("Running TSSpredator for " + prefix)
        out = open(os.path.join(out_path, "log.txt"), "w")
        err = open(os.path.join(out_path, "err.txt"), "w")
        call(["java", "-jar", tsspredator_path,
              config_file], stdout=out, stderr=err)
        out.close()
        err.close()

    def _import_lib(self, libs, wig_folder, project_strain_name,
                    out, gff, program, fasta):
        lib_dict = {"fp": [], "fm": [], "nm": [], "np": []}
        lib_num = 0
        rep_set = set()
        list_num_id = []
        print("Runniun {0} now...".format(program))
        for lib in libs:
            lib_datas = lib.split(":")
            if not lib_datas[0].endswith(".wig"):
                print("Error:Exist a not proper wig files!!")
                sys.exit()
            for wig in os.listdir(wig_folder):
                filename = wig.split("_STRAIN_")
                if (filename[0] == lib_datas[0][:-4]) and (
                        filename[1][:-4] == project_strain_name):
                    lib_datas[0] = wig
            if int(lib_datas[2]) > lib_num:
                lib_num = int(lib_datas[2])
            if lib_datas[3] not in rep_set:
                rep_set.add(lib_datas[3])
            if (lib_datas[1] == "tex") and (lib_datas[4] == "+"):
                lib_dict["fp"].append(self._assign_dict(lib_datas))
            elif (lib_datas[1] == "tex") and (lib_datas[4] == "-"):
                lib_dict["fm"].append(self._assign_dict(lib_datas))
            elif (lib_datas[1] == "notex") and (lib_datas[4] == "+"):
                lib_dict["np"].append(self._assign_dict(lib_datas))
            elif (lib_datas[1] == "notex") and (lib_datas[4] == "-"):
                lib_dict["nm"].append(self._assign_dict(lib_datas))
        for num_id in range(1, lib_num+1):
            out.write("annotation_{0} = {1}\n".format(num_id, gff))
        if program.lower() == "tss":
            self._print_lib(lib_num, lib_dict["fm"], out,
                            wig_folder, "fivePrimeMinus")
            self._print_lib(lib_num, lib_dict["fp"], out,
                            wig_folder, "fivePrimePlus")
        elif program.lower() == "processing_site":
            self._print_lib(lib_num, lib_dict["nm"], out,
                            wig_folder, "fivePrimeMinus")
            self._print_lib(lib_num, lib_dict["np"], out,
                            wig_folder, "fivePrimePlus")
        else:
            print("Error: Wrong program name!!!")
            sys.exit()
        for num_id in range(1, lib_num+1):
            out.write("genome_{0} = {1}\n".format(num_id, fasta))
        for num_id in range(1, lib_num+1):
            list_num_id.append(str(num_id))
        return lib_num, num_id, rep_set, lib_dict, list_num_id

    def _gen_config(self, project_strain_name, args_tss, gff,
                    wig_folder, fasta, config_file):
        master_folder = "MasterTable_" + project_strain_name
        out_path = os.path.join(self.master, master_folder)
        self.helper.check_make_folder(out_path)
        out = open(config_file, "w")
        out.write("TSSinClusterSelectionMethod = HIGHEST\n")
        out.write("allowedCompareShift = 1\n")
        out.write("allowedRepCompareShift = 1\n")
        lib_num, num_id, rep_set, lib_dict, list_num_id = \
            self._import_lib(args_tss.libs, wig_folder, project_strain_name,
                             out, gff, args_tss.program, fasta)
        out.write("idList = ")
        out.write(",".join(list_num_id) + "\n")
        out.write("maxASutrLength = 100\n")
        out.write("maxGapLengthInGene = 500\n")
        out.write("maxNormalTo5primeFactor = {0}\n".format(
                  args_tss.processing_factor))
        out.write("maxTSSinClusterDistance = {0}\n".format(
                  args_tss.cluster + 1))
        out.write("maxUTRlength = {0}\n".format(args_tss.utr_length))
        out.write("min5primeToNormalFactor = {0}\n".format(
                  args_tss.enrichment_factor))
        out.write("minCliffFactor = {0}\n".format(args_tss.factor))
        out.write("minCliffFactorDiscount = {0}\n".format(
                  args_tss.factor_reduction))
        out.write("minCliffHeight = {0}\n".format(args_tss.height))
        out.write("minCliffHeightDiscount = {0}\n".format(
                  args_tss.height_reduction))
        out.write("minNormalHeight = {0}\n".format(args_tss.base_height))
        out.write("minNumRepMatches = {0}\n".format(args_tss.repmatch))
        out.write("minPlateauLength = 0\n")
        out.write("mode = cond\n")
        out.write("normPercentile = 0.9\n")
        if args_tss.program.lower() == "tss":
            self._print_lib(lib_num, lib_dict["nm"], out,
                            wig_folder, "normalMinus")
            self._print_lib(lib_num, lib_dict["np"], out,
                            wig_folder, "normalPlus")
        else:
            self._print_lib(lib_num, lib_dict["fm"], out,
                            wig_folder, "normalMinus")
            self._print_lib(lib_num, lib_dict["fp"], out,
                            wig_folder, "normalPlus")
        out.write("numReplicates = {0}\n".format(len(rep_set)))
        out.write("numberOfDatasets = {0}\n".format(lib_num))
        out.write("outputDirectory = {0}\n".format(out_path))
        for prefix_id in range(len(args_tss.output_prefixs)):
            out.write("outputPrefix_{0} = {1}\n".format(
                      prefix_id + 1, args_tss.output_prefixs[prefix_id]))
        out.write("projectName = {0}\n".format(project_strain_name))
        out.write("superGraphCompatibility = igb\n")
        out.write("texNormPercentile = 0.5\n")
        out.write("writeGraphs = 0\n")
        out.write("writeNocornacFiles = 0\n")
        out.close()

    def _convert_gff(self, prefixs, args_tss):
        for prefix in prefixs:
            out_file = os.path.join(self.gff_outfolder, "_".join([
                           prefix, args_tss.program]) + ".gff")
            gff_f = open(out_file, "w")
            out_path = os.path.join(self.master, "_".join([
                           "MasterTable", prefix]))
            if "MasterTable.tsv" not in os.listdir(out_path):
                print("Error:there is not MasterTable file in {0}".format(
                      out_path))
                print("Please check configuration file.")
            else:
                self.converter.convert_mastertable2gff(
                    os.path.join(out_path, "MasterTable.tsv"),
                    "ANNOgesic", args_tss.program, prefix, out_file)
            gff_f.close()

    def _merge_manual(self, tsss, args_tss):
        self.helper.check_make_folder(os.path.join(os.getcwd(),
                                      self.tmps["tss"]))
        for tss in tsss:
            for gff in os.listdir(args_tss.gffs):
                if (gff[:-4] == tss) and (".gff" in gff):
                    break
            filename = "_".join([tss, args_tss.program]) + ".gff"
            predict = os.path.join(self.gff_outfolder, filename)
            print("Running merge and classify manual ....")
            stat_file = "stat_compare_TSSpredator_manual_{0}.csv".format(tss)
            merge_manual_predict_tss(
                predict, stat_file,
                os.path.join(self.tmps["tss"], filename),
                os.path.join(args_tss.gffs, gff), args_tss)
            shutil.move(stat_file, os.path.join(args_tss.out_folder,
                                                "statistics", tss, stat_file))
        self.helper.move_all_content(self.tmps["tss"],
                                     self.gff_outfolder, [".gff"])
        shutil.rmtree(self.tmps["tss"])

    def _validate(self, tsss, args_tss):
        print("Running validation of annotation....")
        for tss in tsss:
            for gff in os.listdir(args_tss.gffs):
                if (gff[:-4] == tss) and (".gff" in gff):
                    break
            stat_file = os.path.join(
                    self.stat_outfolder, tss,
                    "".join(["stat_gene_vali_", tss, ".csv"]))
            out_cds_file = os.path.join(args_tss.out_folder, "tmp.gff")
            if args_tss.program.lower() == "tss":
                compare_file = os.path.join(self.gff_outfolder,
                                            "_".join([tss, "TSS.gff"]))
            elif args_tss.program.lower() == "processing":
                compare_file = os.path.join(self.gff_outfolder,
                                            "_".join([tss, "processing.gff"]))
            validate_gff(compare_file, os.path.join(args_tss.gffs, gff),
                         stat_file, out_cds_file, args_tss.utr_length,
                         args_tss.program.lower())
            shutil.move(out_cds_file, os.path.join(args_tss.gffs, gff))

    def _compare_ta(self, tsss, args_tss):
        detect = False
        print("Running compare transcript assembly and TSS ...")
        self.multiparser.parser_gff(args_tss.ta_files, "transcript")
        self.multiparser.combine_gff(args_tss.gffs, self.tmps["ta"],
                                     None, "transcript")
        for tss in tsss:
            stat_out = os.path.join(
                    self.stat_outfolder, tss, "".join([
                        "stat_compare_TSS_Transcriptome_assembly_",
                        tss, ".csv"]))
            for ta in os.listdir(self.tmps["ta"]):
                filename = ta.split("_transcript")
                if (filename[0] == tss) and (filename[1] == ".gff"):
                    detect = True
                    break
            compare_file = os.path.join(self.gff_outfolder,
                                        "_".join([tss, "TSS.gff"]))
            if detect:
                stat_ta_tss(os.path.join(self.tmps["ta"], ta), compare_file,
                            stat_out, self.tmps["ta_tss"],
                            self.tmps["tss_ta"], args_tss.fuzzy)
                self.helper.sort_gff(self.tmps["tss_ta"], compare_file)
                self.helper.sort_gff(self.tmps["ta_tss"],
                                     os.path.join(args_tss.ta_files, ta))
                os.remove(self.tmps["tss_ta"])
                os.remove(self.tmps["ta_tss"])
                detect = False

    def _stat_tss(self, tsss, feature):
        print("Running statistaics.....")
        for tss in tsss:
            compare_file = os.path.join(self.gff_outfolder,
                                        "_".join([tss, feature]) + ".gff")
            stat_tsspredator(
                compare_file, feature,
                os.path.join(self.stat_outfolder, tss, "_".join([
                    "stat", feature, "class", tss]) + ".csv"),
                os.path.join(self.stat_outfolder, tss, "_".join([
                    "stat", feature, "libs", tss]) + ".csv"))
            self.helper.move_all_content(os.getcwd(), os.path.join(
                self.stat_outfolder, tss), ["_class", ".png"])
            if os.path.exists(os.path.join(
                    self.stat_outfolder, "TSSstatistics.tsv")):
                shutil.move(
                    os.path.join(
                        self.stat_outfolder, "TSSstatistics.tsv"),
                    os.path.join(
                        self.stat_outfolder, tss, "TSSstatistics.tsv"))
            plot_venn(compare_file, feature)
            self.helper.move_all_content(os.getcwd(), os.path.join(
                self.stat_outfolder, tss), ["_venn", ".png"])

    def _set_gen_config(self, args_tss, input_folder):
        prefixs = []
        detect = False
        for fasta in os.listdir(self.fasta_path):
            for gff in os.listdir(self.gff_path):
                if fasta[:-3] == gff[:-4]:
                    prefix = fasta[:-3]
                    for wig in os.listdir(self.wig_path):
                        filename = wig.split("_STRAIN_")
                        if filename[1][:-4] == prefix:
                            detect = True
                            break
                    if detect:
                        prefixs.append(prefix)
                        config = os.path.join(
                                input_folder,
                                "_".join(["config", prefix]) + ".ini")
                        self._gen_config(
                            prefix, args_tss,
                            os.path.join(self.gff_path, gff), self.wig_path,
                            os.path.join(self.fasta_path, fasta), config)
        return prefixs

    def _merge_wigs(self, wig_folder, prefix, libs):
        self.helper.check_make_folder(os.path.join(os.getcwd(),
                                      self.tmps["tmp"]))
        for wig_file in os.listdir(wig_folder):
            for lib in libs:
                info = lib.split(":")
                if (info[0][:-4] in wig_file) and (info[-1] == "+") and (
                        prefix in wig_file) and (
                        os.path.isfile(os.path.join(wig_folder, wig_file))):
                    Helper().merge_file(
                            os.path.join(wig_folder, wig_file),
                            os.path.join("tmp", "merge_forward.wig"))
                if (info[0][:-4] in wig_file) and (info[-1] == "-") and (
                        prefix in wig_file) and (
                        os.path.isfile(os.path.join(wig_folder, wig_file))):
                    Helper().merge_file(
                            os.path.join(wig_folder, wig_file),
                            os.path.join("tmp", "merge_reverse.wig"))

    def _check_orphan(self, prefixs, wig_folder, args_tss):
        for prefix in prefixs:
            self._merge_wigs(wig_folder, prefix, args_tss.libs)
            tmp_tss = os.path.join(self.tmps["tmp"], "_".join([
                          prefix, args_tss.program + ".gff"]))
            pre_tss = os.path.join(self.gff_outfolder, "_".join([
                          prefix, args_tss.program + ".gff"]))
            check_orphan(pre_tss, os.path.join(
                args_tss.gffs, prefix + ".gff"),
                "tmp/merge_forward.wig", "tmp/merge_reverse.wig", tmp_tss)
            shutil.move(tmp_tss, pre_tss)
        shutil.rmtree("tmp")

    def _remove_files(self, args_tss):
        print("Remove temperary files and folders...")
        self.helper.remove_tmp(args_tss.fastas)
        self.helper.remove_tmp(args_tss.gffs)
        self.helper.remove_tmp(args_tss.wig_folder)
        self.helper.remove_tmp(args_tss.ta_files)
        if "merge_forward.wig" in os.listdir(os.getcwd()):
            os.remove("merge_forward.wig")
        if "merge_reverse.wig" in os.listdir(os.getcwd()):
            os.remove("merge_reverse.wig")

    def _deal_with_overlap(self, out_folder, args_tss):
        if args_tss.overlap_feature.lower() == "both":
            pass
        else:
            print("Comparing TSS and Processing site...")
            if args_tss.program.lower() == "tss":
                for tss in os.listdir(out_folder):
                    if tss.endswith("_TSS.gff"):
                        ref = self.helper.get_correct_file(
                                args_tss.references, "_processing.gff",
                                tss.replace("_TSS.gff", ""), None, None)
                        filter_tss_pro(os.path.join(out_folder, tss),
                                       ref, args_tss.overlap_feature,
                                       args_tss.cluster)
            elif args_tss.program.lower() == "processing_site":
                for tss in os.listdir(out_folder):
                    if tss.endswith("_processing.gff"):
                        ref = self.helper.get_correct_file(
                                args_tss.references, "_TSS.gff",
                                tss.replace("_processing.gff", ""), None, None)
                        filter_tss_pro(os.path.join(out_folder, tss),
                                       ref, args_tss.overlap_feature,
                                       args_tss.cluster)

    def _low_expression(self, args_tss, gff_folder):
        prefix = None
        self._merge_wigs(args_tss.wig_folder, "wig", args_tss.libs)
        for gff in os.listdir(gff_folder):
            if (args_tss.program.lower() == "tss") and (
                    gff.endswith("_TSS.gff")):
                prefix = gff.replace("_TSS.gff", "")
            elif (args_tss.program.lower() == "processing") and (
                    gff.endswith("_processing.gff")):
                prefix = gff.replace("_processing.gff", "")
            if prefix:
                out = open(os.path.join(
                    self.stat_outfolder, prefix, "_".join([
                        "stat", prefix, "low_expression_cutoff.csv"])), "w")
                out.write("\t".join(["strain", "cutoff_coverage"]) + "\n")
                cutoff = filter_low_expression(
                        os.path.join(gff_folder, gff), args_tss,
                        "tmp/merge_forward.wig", "tmp/merge_reverse.wig",
                        "tmp/without_low_expression.gff")
                out.write("\t".join([prefix, str(cutoff)]) + "\n")
                os.remove(os.path.join(gff_folder, gff))
                shutil.move("tmp/without_low_expression.gff",
                            os.path.join(gff_folder, gff))
                prefix = None
        out.close()

    def run_tsspredator(self, args_tss):
        input_folder = os.path.join(args_tss.out_folder, "configs")
        for gff in os.listdir(args_tss.gffs):
            if gff.endswith(".gff"):
                self.helper.check_uni_attributes(os.path.join(
                                                 args_tss.gffs, gff))
        self.helper.check_make_folder(self.gff_outfolder)
        self.multiparser.parser_fasta(args_tss.fastas)
        self.multiparser.parser_gff(args_tss.gffs, None)
        self.multiparser.parser_wig(args_tss.wig_folder)
        prefixs = self._set_gen_config(args_tss, input_folder)
        for prefix in prefixs:
            out_path = os.path.join(
                    self.master, "_".join(["MasterTable", prefix]))
            config_file = os.path.join(
                    input_folder, "_".join(["config", prefix]) + ".ini")
            self._start_to_run(args_tss.tsspredator_path, config_file,
                               out_path, prefix)
            if os.path.exists(os.path.join(out_path, "TSSstatistics.tsv")):
                shutil.move(os.path.join(out_path, "TSSstatistics.tsv"),
                            os.path.join(
                                self.stat_outfolder, "TSSstatistics.tsv"))
        if args_tss.program.lower() == "processing_site":
            args_tss.program = "processing"
        self._convert_gff(prefixs, args_tss)
        if args_tss.check_orphan:
            print("checking the orphan TSS...")
            self._check_orphan(prefixs,
                               os.path.join(args_tss.wig_folder, "tmp"),
                               args_tss)
        self.multiparser.combine_gff(args_tss.gffs, self.gff_outfolder,
                                     None, args_tss.program)
        datas = []
        for gff in os.listdir(self.gff_outfolder):
            if gff.endswith(".gff"):
                gff_folder = gff.replace("".join(["_", args_tss.program,
                                                  ".gff"]), "")
                self.helper.check_make_folder(
                     os.path.join(self.stat_outfolder, gff_folder))
                datas.append(gff_folder)
        if args_tss.remove_low_expression is not None:
            self._low_expression(args_tss, self.gff_outfolder)
        if args_tss.manual is not None:
            self.multiparser.combine_wig(args_tss.gffs, self.wig_path,
                                         None, args_tss.libs)
            self._merge_manual(datas, args_tss)
        self._deal_with_overlap(self.gff_outfolder, args_tss)
        if args_tss.stat:
            self._stat_tss(datas, args_tss.program)
        if args_tss.validate:
            self._validate(datas, args_tss)
        if args_tss.ta_files is not None:
            self._compare_ta(datas, args_tss)
        self._remove_files(args_tss)


"""
i = 0
numbers = []

while i < 6:
    print("At the top i is %d" % i)
    numbers.append(i)

    i = i + 1
    print("Numbers now: ", numbers)
    print("At the bottom i is %d" % i)

print("The numbers: ")
for num in numbers:
    print(num)
"""

n = input("How many numbers do you want your list to go up to?\n> ")
count = input("How much do you want it to incrememnt by?\n> ")

"""
def list_of_numbers(n, count):
    i = 0
    n = int(n)
    count = int(count)
    numbers = []
    while i < n:
        print("At the top, i is %d" % i)
        numbers.append(i)
        i += count
        print("Numbers now: ", numbers)
        print("At the bottom, i is %d" % i)
"""

def list_of_numbers(n, count):
    n = int(n)
    count = int(count)
    numbers = []
    for i in range(0, n, count):
          print("At the top, i is %d" % i)
          numbers.append(i)
          print("Numbers now: ", numbers)
          print("At the bottom, i is %d" % i)

list_of_numbers(n, count)


from modules.controllers.analysis_text import text_cut_word
handle_data_direct = lambda ids_, created_ats, texts, sources, favorite_counts, langs: {
    "id": ids_,
    "created_at": created_ats,
    "text": texts,
    "source": sources,
    'lang': langs,
    'favorite_count': favorite_counts
}

handle_tweets_data = lambda created_ats, ids, sources, texts, langs, favorite_counts, retweet_count, geos, hashtag: {
    "id": ids,
    "created_at": created_ats,
    "text": texts,
    "lang": langs,
    "source": sources,
    "favorite_count": favorite_counts,
    "retweet_count": retweet_count,
    "geo": geos,
    'hashtag': hashtag
}

def handle_user_timeline_data(datas):
    size = len(datas)
    tempp = {
        "id": [],
        "text": [],
        "name": [],
        'created_at': [],
        'likes': [],
        'retweets': [],
        'screen_name': [],
        'location': []
    }
    for jj in range(size):

        retweet_count = datas[jj]['retweet_count']
        print("==================\n")
        print("TT : ", datas[jj]['text'])
        print("TT : ", datas[jj]['user'])

        print("EE : ", datas[jj]['favorite_count'])
        print("lang : ", datas[jj]['lang'])
        print("==================\n")
        # tempp['replies'].append(datas[jj]['reply_count'])
        tempp['retweets'].append(retweet_count)
        # tempp['geo'].append(geo)
        tempp['likes'].append(datas[jj]['favorite_count'])
        tempp['id'].append(datas[jj]['id'])
        tempp['text'].append((datas[jj]['text']))
        tempp['created_at'].append(datas[jj]['created_at'])
        tempp['screen_name'].append(datas[jj]['user']['screen_name'])
        tempp['name'].append(datas[jj]['user']['name'])
        tempp['location'].append(datas[jj]['user']['location'])
    return tempp


#! /usr/bin/env python
# -*- coding: utf-8 -*-
# __author__ = "Lex"
# Date: 2017/11/30

import sys

from .models import Course_selection,Class_register,Teacher,Feedback_record,Course
from .account import Account

main_run_dic={}
student_run_dic={}
teacher_run_dic={}
admin_run_dic={}

#保留用户登录后信息
user_date = {
    'user_name':None,
    'type':None
}

def make_route(index, dic):
    '''
    界面装饰器
    自动添加界面选项
    :param index: 选项序号
    :param dic: 选项程序名
    :return:
    '''
    def foo(func):
        dic[index]=func
    return foo

def init(num,dic):
    '''
    界面初始化
    :param num: 返回/退出序号
    :param dic: 界面选项保存字典
    :return:
    '''
    def init1(func):
        def init2(*args,**kwargs):
            exit_flag = True
            while exit_flag:
                print('欢迎来到Tesla选课系统'.center(20, '='))
                res = func(*args, **kwargs)
                choice = input('请输入要操作编号: ').strip()
                if choice == num:
                    exit_flag = False
                    continue
                if choice in dic:
                    dic[choice]()
                else:
                    print('输入有误，请重新输入')
                    continue
            return res
        return init2
    return init1

@make_route('1', student_run_dic)
def select_course():
    '''
    学生视图
    1 选课
    :return:
    '''
    course_list = []
    select_list = []
    back_flag = True
    while back_flag:
        filter_courses = Course.get_objs()      #获取所有课程信息
        course_record = Course_selection.get_obj_by_datename(user_date['user_name'])    #通过用户名获取选课列表
        num = 1                                                                         #的信息
        print('现有课程如下'.center(20,'='))
        for filter_course in filter_courses:
            print('''\n%s. %s'''%(num,filter_course.name))
            course_list.append(filter_course.name)              #打印出的课程加入到列表中
            num += 1
        choice = input('请输入课程名或b返回： ').strip()
        if choice == 'b':
            back_flag = False
            continue

        if choice in course_list:
            if course_record:                                   #选过课
                Course_selection.remove(course_record.id)       #覆盖删除
                select_list = list(course_record.class_name)    #选课列表重新赋值
            select_list.append(choice)
            select_list = set(select_list)                      #集合去重
            obj_select_course = Course_selection(user_date['user_name'], select_list)
            obj_select_course.save()                            #将新的选课信息写入
            print('选课成功')
        else:
            print('输入有误')
            continue

@make_route('2', student_run_dic)
def take_course_selection():
    '''
    学生视图
    2 查看已选课程
    :return:
    '''
    course_record = Course_selection.get_obj_by_datename(user_date['user_name'])    #通过名字获取选课信息
    # course_record = Course_selection.get_obj_by_user_defined('name',user_date['user_name'])
    num = 1
    print('已选择课程如下'.center(20,'='))
    for course in course_record.class_name:         #遍历用户出选课列表
        print('''\n%s. %s'''%(num,course))
        num += 1

@make_route('3',student_run_dic)
def check_class_record():
    '''
    学生视图
    3 查看上课记录
    :return:
    '''
    teacher_run_dic['2']()          #直接调用老师的查看上课记录 check_course_register()

@make_route('4',student_run_dic)
def feedback():
    '''
    学生视图
    4 评价老师
    :return:
    '''
    res = Feedback_record.creat_feedback()                  #调用评价类中的创建评价功能
    if res:                                                 #res=返回 评价输入的内容
        Feedback_record.increase_reduce_teacher_money(res)  #调用评价类中的增减老师钱功能

@make_route('1',teacher_run_dic)
def create_course_register():
    '''
    老师视图
    1 创建上课记录
    :return:
    '''
    # Class_register.creat_class_register()       #v1直接调用上课记录中的创建功能
    Class_register.creat_class_register()  # v2直接调用上课记录中的创建功能

@make_route('2',teacher_run_dic)
def check_course_register():
    '''
    老师视图
    2 查看上课记录
    :return:
    '''
    while True:
        choice = input('请输入您要查看的课程名或者"b"退出: ').strip()
        course_register_record = Class_register.get_obj_by_datename(choice)     #通过课程名获取课程信息
        if choice == 'b':
            break
        if not course_register_record:
            print('输入有误，请重新输入!')
            continue
        else:
            print('''%s %s %s %s %s'''
                  %(course_register_record.name,
                 course_register_record.student_name,
                 course_register_record.teacher_name,
                 course_register_record.class_part,
                 course_register_record.class_hour))

@make_route('1',admin_run_dic)
def create_course():
    '''
    管理视图
    1 创建、更新课程
    :return:
    '''
    Course.cereat_course()          #直接调用课程类中的创建功能

@make_route('2',admin_run_dic)
def create_course_register():
    '''
    管理视图、更新老师
    :return:
    '''
    Teacher.cereat_teacher()            #直接调用老师的创建功能

@init('5',student_run_dic)
def student_run():
    '''
    学生界面
    1 选课
    2 查看已选课程
    3 查看上课记录
    4 评价老师
    5 返回
    :return:
    '''
    msg='''
    1 选课
    2 查看已选课程
    3 查看上课记录
    4 评价老师
    5 返回
    '''
    print(msg)

@init('3',teacher_run_dic)
def teacher_run():
    '''
    老师界面
    1 创建上课记录
    2 查看上课记录
    3 返回
    :return:
    '''
    msg='''
    1 创建上课记录
    2 查看上课记录
    3 返回
    '''
    print(msg)

@init('3',admin_run_dic)
def admin_run():
    '''
    管理界面
    1 添加/更新课程
    2 添加/更新教师
    3 返回
    :return:
    '''
    msg='''
    1 添加/更新课程
    2 添加/更新教师
    3 返回
    '''
    print(msg)

@make_route('1',main_run_dic)
def student_register():
    '''
    学生注册
    调用账户模块的注册功能
    :return:
    '''
    account = Account.register('student')
    user_date['user_name'] = account.name           #注册后重新获取最新的用户信息
    user_date['type'] = account.account_type

@make_route('2',main_run_dic)
def login():
    '''
    账号登录
    调用账户模块的登陆功能
    登录完成后保存登录用户信息
    :return:
    '''
    account = Account.login()
    user_date['user_name'] = account.name           #登陆后重新获取最新的用户信息
    user_date['type'] = account.account_type

def main_run():
    '''
    程序主界面：
    1 学生注册
    2 账号登陆
    3 退出
    :return:
    '''
    msg = '''
    1 学生注册
    2 账号登陆
    3 退出
    '''
    while True:
        print('欢迎来到万里长城永不倒选课系统'.center(20, '='))
        print(msg)
        choice = input('请输入要操作编号: ').strip()
        if choice == '3':
            sys.exit('谢谢使用！')
        if choice in main_run_dic:
            main_run_dic[choice]()
            if user_date['type'] == 'student':      #通过用户信息中的类型判断进入哪个视图
                student_run()
            elif user_date['type'] == 'teacher':
                teacher_run()
            elif user_date['type'] == 'admin':
                admin_run()
        else:
            print('输入有误，请重新输入')
            continue

if __name__ == '__main__':
    main_run()


#!/usr/bin/python

cache = {}

def rotate(number):
	saved = str(number)
	number = str(number)

	if number in cache:
		return cache[number]

	rotations = set()

	for i in range(len(number)):
		number = number[1:] + number[0]
		rotations.add(int(number))

	cache[saved] = frozenset(rotations)
	return rotations

def rotations(number, bound):
	count = 0

	for rotation in rotate(number):
		if number < rotation and rotation <= bound:
			count += 1

	return count

# let's jam!
in_file = open('2012-0-c.in', 'r')
tests = int(in_file.readline())
out_file = open('2012-0-c.out', 'w')

for test in range(1, tests + 1):
	lower, upper = map(int, in_file.readline().strip().split(' '))
	count = 0

	for number in range(lower, upper):
		count += rotations(number, upper)

	output = 'Case #%d: %d\n' % (test, count)
	out_file.write(output)

out_file.close()
in_file.close()


import os

import numpy as np
from sklearn.datasets import load_svmlight_file

from lib.dataset_analysis import features_characteristics, print_characteristics

# name, path, n_observations, n_features
regression_datasets = [
    ('Abalone Train', '../regression/abalone/abalone.trn.bz2', 4177, 8),
]

def iterate_regression_dataset_path():
    for name, path, *args in regression_datasets:
        path = os.path.join(os.path.dirname(__file__), path)
        yield name, path

def iterate_regression_dataset():
    for name, path, n_observations, n_features in regression_datasets:
        path = os.path.join(os.path.dirname(__file__), path)
        x, y = load_svmlight_file(path)
        yield name, x, y, n_observations, n_features

def describe_regression_datasets():
    for name, x, y, *args in iterate_regression_dataset():
        label_mean = np.mean(y)
        label_std = np.std(y)
        label_min = np.min(y)
        label_max = np.max(y)

        characteristics = features_characteristics(x)
        characteristics['label mean'] = '{:.3g}'.format(label_mean)
        characteristics['label std'] = '{:.3g}'.format(label_std)
        characteristics['label min'] = '{:.3g}'.format(label_min)
        characteristics['label max'] = '{:.3g}'.format(label_max)

        print('\n{:}'.format(name))
        print_characteristics(characteristics, html=False)

# describe_regression_datasets()


import numpy as np
import matplotlib.pyplot as plt
import scipy.stats as sp
import math
import random as rm
import NumerosGenerados as ng
import scipy.interpolate as si

# Graficando Binomial negativa
n, p = 4 , 0.2 # parametros de forma
nbinomial = sp.nbinom(n=n, p=p) # Distribución
xLine = np.arange(nbinomial.ppf(0.01),
              nbinomial.ppf(0.99))
fmp = nbinomial.pmf(xLine) # Función de Masa de Probabilidad
plt.plot(xLine, fmp, '--')
plt.vlines(xLine, 0, fmp, colors='b', lw=5, alpha=0.5)
plt.title('Distribución Binomial Negativa')
plt.ylabel('probabilidad')
plt.xlabel('valores')
plt.show()
print("Media: ", np.mean(xLine))
print("Desvio: ", np.sqrt(np.var(xLine)))
print("Varianza: ", np.var(xLine))

#Naylor

#----------Naylor----------
cant=100000
randomGCL = ng.generarNumeros(cant)

#q = probabilidad de fracaso.

def nBinomial(k, q):
    pascales= []
    qr = math.log(q)
    for j in range (cant):
        tr=1.0
        for i in range(k):
            r = rm.choice(randomGCL)
            tr = tr * r
        nx = math.log(tr)//qr
        pascales.append(nx)
    unicos, cuenta = np.unique(pascales, return_counts=True)
    frec = np.array(cuenta/cant)
    print("Media: ", np.mean(pascales))
    print("Desvio: ", np.sqrt(np.var(pascales)))
    print("Varianza: ", np.var(pascales))
    plt.title("Distribucion Binomial Negativa (Pascal)")
    xnew=np.linspace(unicos.min(),unicos.max(),300)
    spl = si.make_interp_spline(unicos,frec, k=3)
    frec_suavizada=spl(xnew)
    plt.xlim(0,50)
    plt.plot(xLine, fmp, '--',color = "violet")
    plt.vlines(xLine, 0, fmp, colors='black', lw=5, alpha=0.5,linestyles='dashed')

    plt.plot(xnew,frec_suavizada,'--',color="brown")
    plt.bar(unicos, frec, width=0.2, alpha = 0.7)
    plt.show()

#int(n-(n*(1-p)))

nBinomial(n,1-p)


# -*- coding: utf-8 -*-

# Copyright (c) 2016-2019 by University of Kassel and Fraunhofer Institute for Energy Economics
# and Energy System Technology (IEE), Kassel. All rights reserved.

import os
import pytest
import copy

import pandas as pd
import pandapower as pp
import pandapower.topology as top
from pandapower.test.toolbox import assert_net_equal, create_test_network, tempdir, net_in
from pandapower.io_utils import collect_all_dtypes_df, restore_all_dtypes
import pandapower.networks as nw
from pandapower.io_utils import PPJSONEncoder, PPJSONDecoder
import json
import numpy as np

def test_pickle(net_in, tempdir):
    filename = os.path.join(tempdir, "testfile.p")
    pp.to_pickle(net_in, filename)
    net_out = pp.from_pickle(filename)
    assert_net_equal(net_in, net_out)

def test_excel(net_in, tempdir):
    filename = os.path.join(tempdir, "testfile.xlsx")
    pp.to_excel(net_in, filename)
    net_out = pp.from_excel(filename)
    assert_net_equal(net_in, net_out)

    # test in user_pf_options are equal
    pp.set_user_pf_options(net_in, tolerance_mva=1e3)
    pp.to_excel(net_in, filename)
    net_out = pp.from_excel(filename)
    assert_net_equal(net_in, net_out)
    assert net_out.user_pf_options == net_in.user_pf_options

def test_json_basic(net_in, tempdir):
    # tests the basic json functionality with the encoder/decoder classes
    filename = os.path.join(tempdir, "testfile.json")
    with open(filename, 'w') as fp:
        json.dump(net_in, fp, cls=PPJSONEncoder)

    with open(filename) as fp:
        net_out = json.load(fp, cls=PPJSONDecoder)
        pp.convert_format(net_out)

    assert_net_equal(net_in, net_out)

def test_json(net_in, tempdir):
    filename = os.path.join(tempdir, "testfile.json")
    try:
        net_geo = copy.deepcopy(net_in)
        # make GeodataFrame
        from shapely.geometry import Point, LineString
        from fiona.crs import from_epsg
        import geopandas as gpd

        for tab in ('bus_geodata', 'line_geodata'):
            if tab == 'bus_geodata':
                geometry = net_geo[tab].apply(lambda x: Point(x.x, x.y), axis=1)
            else:
                geometry = net_geo[tab].coords.apply(LineString)
            net_geo[tab] = gpd.GeoDataFrame(net_geo[tab], geometry=geometry, crs=from_epsg(4326))

        pp.to_json(net_geo, filename)
        net_out = pp.from_json(filename)
        assert_net_equal(net_geo, net_out)
    except (NameError, ImportError):
        pass

    # check if restore_all_dtypes works properly:
    net_in.line['test'] = 123
    net_in.res_line['test'] = 123
    pp.to_json(net_in, filename)
    net_out = pp.from_json(filename)
    assert_net_equal(net_in, net_out)

def test_type_casting_json(net_in, tempdir):
    filename = os.path.join(tempdir, "testfile.json")
    net_in.sn_kva = 1000
    pp.to_json(net_in, filename)
    net = pp.from_json(filename)
    assert_net_equal(net_in, net)

def test_sqlite(net_in, tempdir):
    filename = os.path.join(tempdir, "testfile.db")
    pp.to_sqlite(net_in, filename)
    net_out = pp.from_sqlite(filename)
    assert_net_equal(net_in, net_out)

def test_convert_format():  # TODO what is this thing testing ?
    net = pp.from_pickle(os.path.join(pp.pp_dir, "test", "api", "old_net.p"))
    pp.runpp(net)
    assert net.converged

def test_restore_all_dtypes():
    net = create_test_network()
    pp.runpp(net)
    net['res_test'] = pd.DataFrame(columns=['test'], data=[1, 2, 3])
    net['test'] = pd.DataFrame(columns=['test'], data=[1, 2, 3])
    net.line['test'] = 123
    net.res_line['test'] = 123
    net.bus['test'] = 123
    net.res_bus['test'] = 123
    net.res_load['test'] = 123
    dtdf = collect_all_dtypes_df(net)
    restore_all_dtypes(net, dtdf)

def test_to_json_dtypes(tempdir):
    filename = os.path.join(tempdir, "testfile.json")
    net = create_test_network()
    pp.runpp(net)
    net['res_test'] = pd.DataFrame(columns=['test'], data=[1, 2, 3])
    net['test'] = pd.DataFrame(columns=['test'], data=[1, 2, 3])
    net.line['test'] = 123
    net.res_line['test'] = 123
    net.bus['test'] = 123
    net.res_bus['test'] = 123
    net.res_load['test'] = 123
    pp.to_json(net, filename)
    net1 = pp.from_json(filename)
    assert_net_equal(net, net1)

def test_json_encoding_decoding():
    net = nw.mv_oberrhein()
    net.tuple = (1, "4")
    net.mg = top.create_nxgraph(net)
    s = set(['1', 4])
    t = tuple(['2', 3])
    f = frozenset(['12', 3])
    a = np.array([1., 2.])
    d = {"a": net, "b": f}
    json_string = json.dumps([s, t, f, net, a, d], cls=PPJSONEncoder)
    s1, t1, f1, net1, a1, d1 = json.loads(json_string, cls=PPJSONDecoder)

    assert s == s1
    assert t == t1
    assert f == f1
    assert net.tuple == net1.tuple
    assert np.allclose(a, a1)

    # TODO line_geodata isn't the same since tuples inside DataFrames are converted to lists (see test_json_tuple_in_dataframe)
    assert pp.nets_equal(net, net1, exclude_elms=["line_geodata"])
    assert pp.nets_equal(d["a"], d1["a"], exclude_elms=["line_geodata"])
    assert d["b"] == d1["b"]
    assert_graphs_equal(net.mg, net1.mg)

def assert_graphs_equal(mg1, mg2):
    edge1 = mg1.edges(data=True)
    edge2 = mg2.edges(data=True)
    for (u, v, data), (u1, v1, data1) in zip(sorted(edge1), sorted(edge2)):
        assert u == u1
        assert v == v1
        if "json_id" in data1:
            del data1["json_id"]
        if "json_key" in data1:
            del data1["json_key"]
        assert data == data1

@pytest.mark.xfail
def test_json_tuple_in_pandas():
    s = pd.Series()
    s["test"] = [(1, 2), (3, 4)]
    json_string = json.dumps(s, cls=PPJSONEncoder)
    s1 = json.loads(json_string, cls=PPJSONDecoder)
    assert (type(s["test"][0]) == type(s1["test"][0]))

if __name__ == "__main__":
    pytest.main(["test_file_io.py", "-x"])


from flask import Flask, render_template
import requests
#import json

app = Flask(__name__)

@app.route('/')
def index():
    resp = requests.get(
        url='http://127.0.0.1:9080/crawl.json?start_requests=false&spider_name=games'
    ).json()

    #items = json.dumps(resp.get('items'))
    items = resp.get('items')
    return render_template('index.html', games=items)

if __name__ == '__main__':
    app.run(debug=True)


import base64
import StringIO
import sys

with open(sys.argv[1], "rb") as tgzIn:
    base64Out = StringIO.StringIO()
    base64.encode(tgzIn, base64Out)
    base64Data = base64Out.getvalue()
    base64Out.close()

code = """
import base64
import io
import tarfile

tgzData = base64.decodestring(\"\"\"
""" + base64Data + """\"\"\")

with io.BytesIO(tgzData) as tgzIn:
    tarFile = tarfile.open(fileobj=tgzIn, mode='r:gz')
    tarFile.extractall()
"""

with open(sys.argv[2], "w") as f:
    f.write(code)


'''
Created on May 16, 2016

@author: peter
'''
from collections import Counter
from collections import defaultdict
import gzip
from multiprocessing import Pool
import re
import time
import operator

import numpy

import sys
import os

sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))

from anton import config
from anton.feature_extraction import base
from anton.util import util

WEBT1_PATTERN_COUNT_FILE = "webt1_pattern_count.txt"

class PosPatternsExtractor(base.FeatureExtractor):

    def title(self):
        return "POS-Patterns"

    def feature_col_headers(self):
        return self.col_headers()

    def col_headers(self):
        return ["bestPattern" + str(i) for i in range(1, 1000)]

    def init(self, pos_to_df):
        return

    def extract_features_pos(self, pos_to_df):
        "This is just a dummy extractor. Pattern features have to be extracted with the Spark project."
        return

    def finish(self, df):
        return

class GoogleNGramPatternExtractor(base.FeatureExtractor):

    def __init__(self):

        self.num_sentences = 95119665584

        self.web1t_dir = config.path_to_res("corpus_dir_web1t")

        self.pattern_length_to_start_ngram_to_gm_filename = defaultdict(dict)
        self.pattern_length_to_gm_filenames = defaultdict(list)

        ngram_dir = self.web1t_dir + "en/gz/"
        for pattern_length in range(3, 6):
            index_filename = ngram_dir + str(pattern_length) + "gms/" + str(pattern_length) + "gm.idx"
            for line in open(index_filename):
                parts = line.split("\t")
                gm_filename = parts[0]
                start_ngram = parts[1].rstrip()
                self.pattern_length_to_gm_filenames[pattern_length].append(gm_filename)
                self.pattern_length_to_start_ngram_to_gm_filename[pattern_length][start_ngram] = gm_filename

        self.pattern_to_count = Counter()
        self.pattern_length_to_patterns = {}
        for pattern_length in range(3, 6):
            self.pattern_length_to_patterns[pattern_length] = []
        self.pattern_to_line_index = Counter()
        self.pattern_to_instance_distance = Counter()
        i = 1
        for line in open(config.ws_path + "/AntonSpk/" + WEBT1_PATTERN_COUNT_FILE):
            parts = line.split('\t')
            pattern = parts[0]
            count = int(parts[1])
            pattern_length = len(pattern.split())
            self.pattern_length_to_patterns[pattern_length].append(pattern)
            self.pattern_to_count[pattern] = count
            self.pattern_to_line_index[pattern] = i

            tokens = pattern.split()
            wildcard_index_1 = tokens.index("(\w+)")
            self.pattern_to_instance_distance[pattern] = tokens[wildcard_index_1 + 1:].index("(\w+)") + 2

            i += 1

    def title(self):
        return "Google n-gram patterns"

    def feature_col_headers(self):
        return ["web1t_pattern_" + str(line_index) + "_lmi" for _, line_index in self.pattern_to_line_index.items()]

    def col_headers(self):
        instance_count_col_headers = ["web1t_instance_" + str(pattern_length) + "_gram_freq" for pattern_length, _ in self.pattern_length_to_patterns.items()]
        cooc_col_headers = ["web1t_pattern_" + str(line_index) + "_freq" for _, line_index in self.pattern_to_line_index.items()]
        return self.feature_col_headers() + instance_count_col_headers + cooc_col_headers

    def init(self, pos_to_df):
        return

    def extract_features_pos(self, pos_to_df):

        pattern_to_instance_to_count = defaultdict(Counter)

        for pattern_length, patterns in self.pattern_length_to_patterns.items():

            pattern_to_instance_to_ngrams, ngrams = self.prepare_ngrams(pattern_length, patterns, pos_to_df)

            ngram_to_count, instance_to_count = self.determine_ngram_and_instance_counts(pattern_length, ngrams, pos_to_df)

            for pattern in patterns:

                for pos in util.POS:
                    df = pos_to_df[pos]

                    for instance, _ in df.iterrows():
                        pattern_instance_ngrams = pattern_to_instance_to_ngrams[pattern][instance]
                        count = ngram_to_count[pattern_instance_ngrams[0]] + ngram_to_count[pattern_instance_ngrams[1]]
                        pattern_to_instance_to_count[pattern][instance] = count

            for pos in util.POS:
                df = pos_to_df[pos]
                df["web1t_instance_" + str(pattern_length) + "_gram_freq"] = df.index.map(lambda instance: instance_to_count[instance])

        for pos in util.POS:
            df = pos_to_df[pos]

            for pattern_length, patterns in self.pattern_length_to_patterns.items():

                for pattern in patterns:

                    pattern_line_index = self.pattern_to_line_index[pattern]

                    cooc_col_header = "web1t_pattern_" + str(pattern_line_index) + "_freq"
                    df[cooc_col_header] = df.index.map(lambda instance: pattern_to_instance_to_count[pattern][instance])

                    numerator = self.num_sentences * df[cooc_col_header]
                    denominator = df["web1t_instance_" + str(self.pattern_to_instance_distance[pattern]) + "_gram_freq"] * self.pattern_to_count[pattern]

                    pattern_lmi_col_header = "web1t_pattern_" + str(pattern_line_index) + "_lmi"
                    df[pattern_lmi_col_header] = df[cooc_col_header] * numpy.log(numerator / denominator)
                    df[pattern_lmi_col_header].replace(to_replace=numpy.inf, value=0, inplace=True)
                    df[pattern_lmi_col_header].fillna(value=-sys.maxsize, inplace=True)
        return

    def prepare_ngrams(self, pattern_length, patterns, pos_to_df):

        pattern_to_instance_to_ngrams = defaultdict(lambda: defaultdict(list))
        ngrams = []

        for pattern in patterns:

            for pos in util.POS:
                df = pos_to_df[pos]

                for instance, _ in df.iterrows():

                    parts = instance.split()
                    word_1, word_2 = parts[0], parts[1]
                    ngram_w1_w2 = pattern.replace("(\w+)", word_1, 1).replace("(\w+)", word_2, 1)
                    ngram_w2_w1 = pattern.replace("(\w+)", word_2, 1).replace("(\w+)", word_2, 1)
                    pattern_instance_ngrams = [ngram_w1_w2, ngram_w2_w1]
                    ngrams.extend(pattern_instance_ngrams)
                    pattern_to_instance_to_ngrams[pattern][instance].extend(pattern_instance_ngrams)

        ngrams.sort()

        return pattern_to_instance_to_ngrams, ngrams

    def map_gm_files_to_ngrams(self, pattern_length, ngrams):

        start_ngrams = sorted([start_ngram for start_ngram, _ in self.pattern_length_to_start_ngram_to_gm_filename[pattern_length].items()])

        num_gm_files = len(start_ngrams)

        file_to_ngrams = defaultdict(list)
        i = j = 0
        while i < num_gm_files - 1:

            start_ngram = start_ngrams[i]
            next_start_ngram = start_ngrams[i + 1]

            while j < len(ngrams) and ngrams[j] < next_start_ngram:

                if ngrams[j] >= start_ngram:
                    gm_filename = self.pattern_length_to_start_ngram_to_gm_filename[pattern_length][start_ngram]
                    file_to_ngrams[gm_filename].append(ngrams[j])
                j += 1

            i += 1

        last_gm_filename = self.pattern_length_to_start_ngram_to_gm_filename[pattern_length][start_ngrams[num_gm_files - 1]]
        file_to_ngrams[last_gm_filename].extend(ngrams[i:])

        return file_to_ngrams

    def determine_ngram_and_instance_counts(self, pattern_length, ngrams, pos_to_df):

        pair_to_count = Counter()
        ngram_to_count = Counter()
        instance_to_count = Counter()

        pairs = set()
        for pos in util.POS:
            df = pos_to_df[pos]
            for instance, _ in df.iterrows():
                parts = instance.split()
                pairs.add(util.order_pair((parts[0], parts[1])))

        file_to_ngrams = self.map_gm_files_to_ngrams(pattern_length, ngrams)

        ngram_dir = self.web1t_dir + "en/gz/" + str(pattern_length) + "gms/"

        filenames = self.pattern_length_to_gm_filenames[pattern_length]
        num_files = len(filenames)
        for i in range(0, num_files, 4):

            pool = Pool()

            result1 = pool.apply_async(determine_ngram_and_pair_counts_in_file, [filenames[i], ngram_dir, pairs, file_to_ngrams[filenames[i]]])
            if not i + 1 > num_files - 1:
                result2 = pool.apply_async(determine_ngram_and_pair_counts_in_file, [filenames[i + 1], ngram_dir, pairs, file_to_ngrams[filenames[i + 1]]])
            if not i + 2 > num_files - 1:
                result3 = pool.apply_async(determine_ngram_and_pair_counts_in_file, [filenames[i + 2], ngram_dir, pairs, file_to_ngrams[filenames[i + 2]]])
            if not i + 3 > num_files - 1:
                result4 = pool.apply_async(determine_ngram_and_pair_counts_in_file, [filenames[i + 3], ngram_dir, pairs, file_to_ngrams[filenames[i + 3]]])

            answer1 = result1.get(timeout=10000)
            if not i + 1 > num_files - 1:
                answer2 = result2.get(timeout=10000)
                answer1 = tuple(map(operator.add, answer2, answer1))
            if not i + 2 > num_files - 1:
                answer3 = result3.get(timeout=10000)
                answer1 = tuple(map(operator.add, answer3, answer1))
            if not i + 3 > num_files - 1:
                answer4 = result4.get(timeout=10000)
                answer1 = tuple(map(operator.add, answer4, answer1))

            pool.close()

            ngram_to_count += answer1[0]
            pair_to_count += answer1[1]

        for pair, count in pair_to_count.items():
            instance_to_count[pair[0] + " " + pair[1]] = count

        return ngram_to_count, instance_to_count

    def finish(self, df):
        return

def determine_ngram_and_pair_counts_in_file(filename, ngram_dir, pairs, ngrams):

    ngram_to_count = Counter()
    pair_to_count = Counter()
    for pair in pairs:
        pair_to_count[pair] = 0

    i = 0
    num_ngrams = len(ngrams)

    start_time = time.time()
    print("Processing " + filename)

    with gzip.open(ngram_dir + filename, 'r') as file:

        for line in file:
            parts = line.decode().split('\t')
            ngram = parts[0]

            tokens = ngram.split()
            try:
                pair = util.order_pair((tokens[0], tokens[-1]))

                if pair in pair_to_count:
                    pair_to_count[pair] += int(parts[1])
            except IndexError:
                pass

            if ngrams and i < num_ngrams:
                if ngram == ngrams[i]:
                    ngram_to_count[ngram] = int(parts[1])
                    i += 1
                elif ngram > ngrams[i]:
                    i += 1

    print("Took" + str(time.time() - start_time) + " to process one file.")

    return ngram_to_count, pair_to_count

def count_patterns_in_file(filename, ngram_dir, patterns, pattern_to_tokens, pattern_to_regex):

        pattern_to_count = Counter()

        print("Processing: " + filename)

        start_time = time.time()
        with gzip.open(ngram_dir + '/' + filename, 'r') as file:
            for line in file:

                parts = line.decode().split('\t')
                ngram = parts[0]

                ngram_tokens = set(ngram.split())

                for pattern in patterns:
                    if pattern_to_tokens[pattern].issubset(ngram_tokens):
                        if pattern_to_regex[pattern].match(ngram):
                            pattern_to_count[pattern] += int(parts[1])

        print("Took" + str(time.time() - start_time) + " to process one file.")

        return pattern_to_count

def prepare_web1t_patterns():

    web1t_dir = config.path_to_res("corpus_dir_web1t")

    ws_path = config.ws_path

    pattern_ranking_lines = []
    for pos in util.POS:
        lines = open(ws_path + "/AntonSpk/" + pos.tag + "_pattern_ranking.txt").readlines()
        pattern_ranking_lines.extend(lines)

    length_to_pattern_to_count = {}
    for pattern_length in range(2, 6):
        length_to_pattern_to_count[pattern_length] = Counter()

    pattern_to_regex = {}
    pattern_to_tokens = {}

    for line in pattern_ranking_lines:
        parts = line.split("\t")
        raw_pattern = parts[0]
        pattern = re.sub(r"/[^ ]*", "", raw_pattern)

        pattern_length = len(pattern.split())
        if pattern_length > 5:
            continue

        length_to_pattern_to_count[pattern_length][pattern] = 0
        pattern_to_regex[pattern] = re.compile(pattern)
        tokens = set(pattern.split())
        tokens.remove("(\w+)")
        pattern_to_tokens[pattern] = tokens

    for pattern_length, pattern_to_count in length_to_pattern_to_count.items():

        ngram_dir = web1t_dir + "en/gz/" + str(pattern_length) + "gms/"

        filenames = [filename for filename in os.listdir(ngram_dir) if filename.endswith("gz")]

        patterns = [pattern for pattern in pattern_to_count.keys()]

        num_files = len(filenames)
        for i in range(0, num_files, 4):

            pool = Pool()

            result1 = pool.apply_async(count_patterns_in_file, [filenames[i], ngram_dir, patterns, pattern_to_tokens, pattern_to_regex])
            if not i + 1 > num_files - 1:
                result2 = pool.apply_async(count_patterns_in_file, [filenames[i + 1], ngram_dir, patterns, pattern_to_tokens, pattern_to_regex])
            if not i + 2 > num_files - 1:
                result3 = pool.apply_async(count_patterns_in_file, [filenames[i + 2], ngram_dir, patterns, pattern_to_tokens, pattern_to_regex])
            if not i + 3 > num_files - 1:
                result4 = pool.apply_async(count_patterns_in_file, [filenames[i + 3], ngram_dir, patterns, pattern_to_tokens, pattern_to_regex])

            answer1 = result1.get(timeout=10000)
            if not i + 1 > num_files - 1:
                answer2 = result2.get(timeout=10000)
                answer1 = answer2 + answer1
            if not i + 2 > num_files - 1:
                answer3 = result3.get(timeout=10000)
                answer1 = answer3 + answer1
            if not i + 3 > num_files - 1:
                answer4 = result4.get(timeout=10000)
                answer1 = answer4 + answer1

            pool.close()

            pattern_to_count += answer1

    with open(config.ws_path + "/AntonSpk/" + WEBT1_PATTERN_COUNT_FILE, 'w') as webt1_file:

        for _, pattern_to_count in length_to_pattern_to_count.items():
            for pattern, count in pattern_to_count.items():

                webt1_file.write(pattern + '\t' + str(count) + '\n')

if __name__ == "__main__":

    prepare_web1t_patterns()


"""
Given an array of 2n integers, your task is to group these integers into n pairs of integer,
say (a1, b1), (a2, b2), ..., (an, bn) which makes sum of min(ai, bi) for all i from 1 to n as large as possible

Input: [1,4,3,2]

Output: 4
Explanation: n is 2, and the maximum sum of pairs is 4 = min(1, 2) + min(3, 4).

"""

class Solution:
    def arrayPairSum(self, nums):
        arr = sorted(nums)
        return sum(arr[0::2])

if __name__ == "__main__":
    s = Solution()
    print(s.arrayPairSum(nums=[1,4,3,2]))