import tkinter as tk
from tkinter import ttk
from tkinter import font, colorchooser, filedialog, messagebox
import os

main_application = tk.Tk()
main_application.geometry('800x600')
main_application.title('vpad text Editor')
main_application.wm_iconbitmap('icon.ico')

############################# main menu #####################################
############################ main menu ending ###############################

#file

main_menu = tk.Menu()

#file icons
new_icon = tk.PhotoImage(file='icons2/new.png')
open_icon = tk.PhotoImage(file='icons2/open.png')
save_icon = tk.PhotoImage(file='icons2/save.png')
save_as_icon = tk.PhotoImage(file='icons2/save_as.png')
exit_icon = tk.PhotoImage(file='icons2/exit.png')

file = tk.Menu(main_menu, tearoff=False) #True leads to separate the menu bar.

####Edit
##edit icons
copy_icon = tk.PhotoImage(file='icons2/copy.png')
paste_icon = tk.PhotoImage(file='icons2/paste.png')
copy_icon = tk.PhotoImage(file='icons2/copy.png')
cut_icon = tk.PhotoImage(file='icons2/cut.png')
clear_all_icon = tk.PhotoImage(file='icons2/clear_all.png')
find_icon = tk.PhotoImage(file='icons2/find.png')

edit = tk.Menu(main_menu, tearoff=False)

##view icons
tool_bar_icon = tk.PhotoImage(file='icons2/tool_bar.png')
status_bar_icon = tk.PhotoImage(file='icons2/status_bar.png')

view = tk.Menu(main_menu, tearoff=False)

#### Color_Theme
light_default_icon = tk.PhotoImage(file='icons2/light_default.png')
light_plus_icon = tk.PhotoImage(file='icons2/light_plus.png')
dark_icon = tk.PhotoImage(file='icons2/dark.png')
red_icon = tk.PhotoImage(file='icons2/red.png')
monokai_icon = tk.PhotoImage(file='icons2/monokai.png')
night_blue_icon = tk.PhotoImage(file='icons2/night_blue.png')

color_theme = tk.Menu(main_menu, tearoff=False)

theme_choice = tk.StringVar()
color_icons = (light_default_icon,light_plus_icon,dark_icon,red_icon,monokai_icon,night_blue_icon)

color_dict = { #dictionary
	'Light Default': ('#000000' , '#ffffff'), #ff-> text color, ss->background color
	'Light Plus' : ('#474747', '#e0e0e0'),
	'Dark': ('#c4c4c4','#2d2d2d'),
	'Red': ('#2d2d2d','#ffe8e8'),
	'Monokai' :('#d3b774','#474747'),
	'Night Blue':('#ededed','#6b9dc2')
}

###cascade
main_menu.add_cascade(label = 'file', menu=file)
main_menu.add_cascade(label = 'edit', menu=edit)
main_menu.add_cascade(label = 'view', menu=view)
main_menu.add_cascade(label = 'color_theme', menu=color_theme)

############################# tootlbar #####################################

#print(type(tk.font.families()))

tool_bar = ttk.Label(main_application)
tool_bar.pack(side=tk.TOP,fill=tk.X) #Will be placed in the upperside. tk.X for horizontally Fill.

font_tuple = tk.font.families()
font_family = tk.StringVar()
font_box = ttk.Combobox(tool_bar, width=30,textvariable = font_family, state='readonly' )
font_box['values'] = font_tuple
font_box.current(font_tuple.index('Times New Roman'))
font_box.grid(row=0,column=0, padx=5)

##Size Box

size_var  = tk.IntVar()
font_size = ttk.Combobox(tool_bar, width=14, textvariable=size_var,state='readonly')
font_size['values'] = tuple(range(8,81)) # ++ incraese
font_size.current(4)
font_size.grid(row=0,column=1,padx=5)

##bold button
bold_icon = tk.PhotoImage(file='icons2/bold.png')
bold_btn = ttk.Button(tool_bar, image = bold_icon)
bold_btn.grid(row=0,column=2,padx=5)

##italic Button
italic_icon = tk.PhotoImage(file='icons2/italic.png')
italic_btn = ttk.Button(tool_bar, image=italic_icon)
italic_btn.grid(row=0,column=3,padx=5)

##Underline Button
underline_icon = tk.PhotoImage(file='icons2/underline.png')
underline_btn = ttk.Button(tool_bar,image = underline_icon)
underline_btn.grid(row=0, column=4, padx=5)

##Font Color button
font_color_icon = tk.PhotoImage(file='icons2/font_color.png')
font_color_btn = ttk.Button(tool_bar, image=font_color_icon)
font_color_btn.grid(row=0,column=5,padx=5)

##Align left
align_left_icon = tk.PhotoImage(file='icons2/align_left.png')
align_left_btn = ttk.Button(tool_bar, image=align_left_icon)
align_left_btn.grid(row=0,column=6,padx=5)

##Align Center
align_center_icon = tk.PhotoImage(file='icons2/align_center.png')
align_center_btn = ttk.Button(tool_bar, image=align_center_icon)
align_center_btn.grid(row=0, column=7,padx=5)

##Algin Right
align_right_icon = tk.PhotoImage(file='icons2/align_right.png')
align_right_btn = ttk.Button(tool_bar, image=align_right_icon)
align_right_btn.grid(row=0, column=8, padx=5)
############################ toolbar ending ###############################

############################# text editor #####################################
text_editor = tk.Text(main_application)
text_editor.config(wrap='word', relief =tk.FLAT) # if word is getting full then it will start from next line.

scroll_bar = tk.Scrollbar(main_application) # scroll bar will appear in right side.
text_editor.focus_set() # cursor will blink
scroll_bar.pack(side=tk.RIGHT, fill=tk.Y)
text_editor.pack(fill=tk.BOTH, expand=True)
scroll_bar.config(command =text_editor.yview)
text_editor.config(yscrollcommand=scroll_bar.set)

#font family and font size functionality
current_font_family = 'Times New Roman'
current_font_size = 12

def change_font(event = None): #main_application
	global current_font_family
	current_font_family = font_family.get() #which font be given by user is getting by this func
	text_editor.configure(font = (current_font_family,current_font_size) )

def change_fontsize(event=None):
	global current_font_size
	current_font_size = size_var.get() #which font be given by user is getting by this func
	text_editor.configure(font = (current_font_family,current_font_size) )

font_box.bind("<<ComboboxSelected>>",change_font) #Bind the function with combobox.
font_size.bind("<<ComboboxSelected>>", change_fontsize)

################## Buttons Functionality

#bold button functionality
def change_bold():
	text_property = tk.font.Font(font=text_editor['font'])
	if text_property.actual()['weight'] == 'normal':
		text_editor.configure(font = (current_font_family,current_font_size,'bold'))
	if text_property.actual()['weight'] == 'bold':
		text_editor.configure(font = (current_font_family,current_font_size,'normal'))

bold_btn.configure(command=change_bold) #This is a button, that's why we give a command.

##italic functionality
def change_italic():
	text_property = tk.font.Font(font =text_editor['font'])
	if text_property.actual()['slant'] == 'roman':
		text_editor.configure(font = (current_font_family,current_font_size,'italic'))
	if text_property.actual()['slant'] == 'italic':
		text_editor.configure(font = (current_font_family,current_font_size,'normal'))
italic_btn.configure(command=change_italic)

##Underline functionality
def change_underline():
	text_property = tk.font.Font(font =text_editor['font'])
	if text_property.actual()['underline'] == 0:
		text_editor.configure(font = (current_font_family,current_font_size,'underline'))
	if text_property.actual()['underline'] == 1:
		text_editor.configure(font = (current_font_family,current_font_size,'normal'))
underline_btn.configure(command=change_underline)

##font color functionality
def change_font_color():
	color_var = tk.colorchooser.askcolor()
	# print(color_var) #this will show a tuple made with RGB val and a hexa color code.
	text_editor.configure(fg = color_var[1]) #1 because we take the hexa value.

font_color_btn.configure(command=change_font_color)

### Align left functionality
def align_left():
	text_content = text_editor.get(1.0 ,'end') #we will store everything from first to last written in text editor.
	text_editor.tag_config('left', justify = tk.LEFT)
	text_editor.delete(1.0,tk.END)
	text_editor.insert(tk.INSERT, text_content,'left')

align_left_btn.configure(command=align_left)

##Align center functionality
def align_center():
	text_content = text_editor.get(1.0 ,'end') #we will store everything from first to last written in text editor.
	text_editor.tag_config('center', justify = tk.CENTER)
	text_editor.delete(1.0,tk.END)
	text_editor.insert(tk.INSERT, text_content,'center')

align_center_btn.configure(command=align_center)

##right alignment
def align_right():
	text_content = text_editor.get(1.0 ,'end') #we will store everything from first to last written in text editor.
	text_editor.tag_config('right', justify = tk.RIGHT)
	text_editor.delete(1.0,tk.END)
	text_editor.insert(tk.INSERT, text_content,'right')

align_right_btn.configure(command=align_right)

text_editor.configure(font=('Times New Roman',12)) # By deafult will type in Arial.

############################ text editor ending ###############################

#############################  status bar #####################################
status_bar = ttk.Label(main_application, text='Status Bar')
status_bar.pack(side = tk.BOTTOM) #When you know , where to place the thing then use pack otherwise use grid.

text_chnaged = False
def changed(event=None):
	global text_chnaged
	if text_editor.edit_modified():
		text_chnaged = True
		words = len(text_editor.get(1.0, 'end-1c').split()) #If we just use 'end' then they count the newline. so , we need to use 'end-1c'.
		characters = len(text_editor.get(1.0, 'end-1c'))
		##If we want not to count the spaces
		# characters = len(text_editor.get(1.0, 'end-1c').replace(' ','')) #spaces will be replaced .

		status_bar.config(text= f'characters : {characters} Words : {words}')
	text_editor.edit_modified(False)

text_editor.bind('<<Modified>>',changed)

############################ main statur bar ending ###############################

############################# main menu functionality #####################################

url = ''

#### New_File Functionality
def new_file(event=None):
	global url                ###working procedure: If data is present from before then it will be deleted . SO , we are getting new file.
	url = ''
	text_editor.delete(1.0, tk.END)

#### Open Functionality

def open_file(event=None):
	global url
	url = filedialog.askopenfilename( initialdir = os.getcwd(), title='Select File', filetypes = ( ('Text File','*.txt'),('All files','*.*') ) )
	try:
		with open(url, 'r') as fr:
			text_editor.delete(1.0, tk.END)
			text_editor.insert(1.0, fr.read())
	except FileNotFoundError:
		return
	except:
		return
	main_application.title(os.path.basename(url))

##Save file functionality

def save_file(event=None):
	global url
	try:
		if url:
			content = str(text_editor.get(1.0,tk.END))
			with open(url, 'w', encoding = 'utf-8') as fw:
				fw.write(content)
		else:
			url = filedialog.asksaveasfile( mode = 'w', defaultextension='.txt',filetypes = ( ('Text File','*.txt'),('All files','*.*') ) )
			content2 = text_editor.get(1.0,tk.END)
			url.write(content)
			url.close()
	except:
		return
#### Change kore file save kore editor kete dile , pore abar open korle saved jinis thakteche na.

##Save as functionality
def save_as(event=None):
	global url
	try:
		content = text_editor.get(1.0,tk.END)
		url = filedialog.asksaveasfile( mode = 'w', defaultextension='.txt',filetypes = ( ('Text File','*.txt'),('All files','*.*') ) )
		url.write(content)
		url.close()
	except:
		return

##Exit functionality
def exit_func(event=None):
	global url, text_chnaged
	try:
		if text_chnaged:
			mbox = messagebox.askyesnocancel('Warnings','Do you want to save the file ?')
			if mbox is True: # yes , no or cancel
				if url: # If we want to save the existing file
					content = text_editor.get(1.0, tk.END)
					with open(url,'w',encoding='utf-8') as fw:
						fw.write(content)
						main_application.destroy()
				else: # file doeesn't exist from before.
					content2 = str( text_editor.get(1.0, tk.END) )
					url = filedialog.asksaveasfile( mode = 'w', defaultextension='.txt',filetypes = ( ('Text File','*.txt'),('All files','*.*') ) )
					url.write(content2)
					url.close()
					main_application.destroy()
			elif mbox is False:
				main_application.destroy()

		else:
			main_application.destroy()
	except:
		return

##file command

file.add_command(label='New',image=new_icon,compound=tk.LEFT, accelerator='Ctrl+N',command=new_file)
file.add_command(label='open',image=open_icon,compound=tk.LEFT, accelerator='Ctrl+O',command = open_file)
file.add_command(label='Save',image=save_icon,compound=tk.LEFT, accelerator='Ctrl+S',command = save_file)
file.add_command(label='Save as',image=save_as_icon,compound=tk.LEFT, accelerator='Ctrl+Alt+S',command = save_as)
file.add_command(label='Exit',image=exit_icon,compound=tk.LEFT, accelerator='Ctrl+Q',command = exit_func)

##edit command

##find functionality

def find_func(event=None):

	def find():
		word = find_input.get()
		text_editor.tag_remove('match','1.0',tk.END)
		matches = 0
		if  word:
			start_pos = '1.0'
			while True:
				start_pos = text_editor.search(word,start_pos,stopindex=tk.END)
				if not start_pos:
					break
				end_pos = f'{start_pos}+{len(word)}c' ##Bujhini
				text_editor.tag_add('match',start_pos,end_pos)
				matches += 1
				start_pos = end_pos
				text_editor.tag_config('match', foreground ='red', background='yellow')

	def replace():
		word = find_input.get()
		replace_text = replace_input.get()
		content = text_editor.get(1.0,tk.END)
		new_content = content.replace(word, replace_text)
		text_editor.delete(1.0,tk.END)
		text_editor.insert(1.0,new_content)

	find_dialogue = tk.Toplevel()
	find_dialogue.geometry('450x250+500+200')
	find_dialogue.title('Find')
	find_dialogue.resizable(0,0) # can't resize

	##frame
	find_frame = ttk.LabelFrame(find_dialogue, text='Find/Replace')
	find_frame.pack(pady=20)

	##labels
	text_find_label = ttk.Label(find_frame, text ='Find: ')
	text_replace_label = ttk.Label(find_frame,text='Replace')

	##entry
	find_input = ttk.Entry(find_frame, width=30)
	replace_input = ttk.Entry(find_frame, width=30)

	##button
	find_button = ttk.Button(find_frame, text='Find',command=find)
	replace_button  = ttk.Button(find_frame, text='Replace',command=replace)

	##label grid
	text_find_label.grid(row=0, column=0, padx=4, pady=4)
	text_replace_label.grid(row=1, column=0, padx=4, pady=4)

	##entry grid
	find_input.grid(row=0,column=1,padx=4,pady=4)
	replace_input.grid(row=1,column=1,padx=4,pady=4)

	##buttons grid
	find_button.grid(row=2, column=0, padx=8, pady=4)
	replace_button.grid(row=2, column=1, padx=8, pady=4)

	find_dialogue.mainloop()

edit.add_command(label='Copy',image=copy_icon, compound=tk.LEFT, accelerator='Ctrl+C', command=lambda:text_editor.event_generate("<Control c>"))
edit.add_command(label='Paste',image=paste_icon, compound=tk.LEFT, accelerator='Ctrl+V', command=lambda:text_editor.event_generate("<Control v>"))
edit.add_command(label='Cut',image=cut_icon, compound=tk.LEFT, accelerator='Ctrl+X', command=lambda:text_editor.event_generate("<Control x>"))
edit.add_command(label='Clear All',image=clear_all_icon, compound=tk.LEFT, accelerator='Ctrl+Alt+X', command = lambda:text_editor.delete(1.0,tk.END)) ###Keyboard button isn't working
edit.add_command(label='Find',image=find_icon, compound=tk.LEFT, accelerator='Ctrl+F',command = find_func)

##view checkbutton

show_statusbar = tk.BooleanVar()
show_statusbar.set(True)
show_toolbar = tk.BooleanVar()
show_toolbar.set(True)

def hide_toolbar():
	global show_toolbar
	if show_toolbar:
		tool_bar.pack_forget()
		show_toolbar = False
	else:
		text_editor.pack_forget() ### We arrange the whole things . Othewise toolbar will not shown in the previous place
		status_bar.pack_forget()
		tool_bar.pack(side=tk.TOP, fill=tk.X)
		text_editor.pack(fill=tk.BOTH, expand=True)
		status_bar.pack(side=tk.BOTTOM)
		show_toolbar = True

def hide_statusbar():
	global show_statusbar
	if show_statusbar:
		status_bar.pack_forget()
		show_statusbar = False
	else:
	    status_bar.pack(side=tk.BOTTOM)
	    show_statusbar = True

view.add_checkbutton(label='Tool Bar',onvalue = True, offvalue = 0, variable= show_toolbar,image=tool_bar_icon,compound=tk.LEFT,command = hide_toolbar)
view.add_checkbutton(label='Status Bar',onvalue=1,offvalue=False,variable= show_statusbar, image=status_bar_icon,compound=tk.LEFT,command = hide_statusbar)

#color theme

def change_theme():
	chosen_theme = theme_choice.get()
	color_tuple = color_dict.get(chosen_theme)
	fg_color, bg_color = color_tuple[0], color_tuple[1]
	text_editor.config(background=bg_color, fg=fg_color) # fg means foregruound color

count = 0
for i in color_dict:
	color_theme.add_radiobutton(label= i, image=color_icons[count], variable=theme_choice, compound=tk.LEFT, command = change_theme)
	count+=1

main_application.config(menu=main_menu)

############################ main menu functionality ending ###############################

main_application.bind("<Control-n>", new_file)
main_application.bind("<Control-o>", open_file)
main_application.bind("<Control-s>", save_file)
main_application.bind("<Control-Alt-s>", save_as)
main_application.bind("<Control-q>", exit_func)
main_application.bind("<Control-f>", find_func)

main_application.mainloop()


from django.conf.urls import url
from django.views.generic.base import RedirectView
from .views import user_views, db_views, report_views, vps_views, task_views, log_views, views

urlpatterns = {
    #  ====================================================================================
    # 前端界面
    url(r'^index$', views.index, name='index'),
    url(r'^favicon\.ico$', RedirectView.as_view(url='/static/isadmin/images/favicon.ico')),
    # 用户相关
    url(r'^$', user_views.login, name='login'),
    url(r'^login$', user_views.login, name='login'),
    url(r'^register$', user_views.register, name='register'),
    url(r'^retrieve$', user_views.retrieve, name='retrieve'),
    # db管理
    url(r'^pubsoc_list$', db_views.pubsoc_list, name='pubsoc_list'),
    url(r'^prisoc_list$', db_views.prisoc_list, name='prisoc_list'),
    url(r'^malicious_domains_list$', db_views.malicious_domains_list, name='malicious_domains_list'),
    url(r'^snapshot_list$', db_views.snapshot_list, name='snapshot_list'),
    url(r'^suspicious_records_list$', db_views.suspicious_records_list, name='suspicious_record_list'),
    url(r'^pocr_list$', db_views.pocr_list, name='pocr_list'),
    url(r'^dcbp_list$', db_views.dcbp_list, name='dcbp_list'),
    # 快照展示
    url(r'^show_snapshot$', views.show_snapshot, name='show_snapshot'),
    url(r'^show_snapshot/(?P<id>[0-9]+)$', views.show_snapshot, name='show_snapshot'),
    # vps监控
    url(r'^vps_monitor$', vps_views.vps_monitor, name='vps_monitor'),
    url(r'^vps_monitor_reload$', vps_views.vps_monitor_reload, name='vps_monitor_reload'),
    url(r'^vps_detail$', vps_views.vps_detail, name='vps_detail'),
    url(r'^cpu_chart$', vps_views.cpu_chart, name='cpu_chart'),
    url(r'^memory_chart$', vps_views.memory_chart, name='memory_chart'),
    url(r'^disks_chart$', vps_views.disks_chart, name='disks_chart'),
    # 数据报表
    url(r'^report/redirect_records$', report_views.redirect_records, name='redirect_records'),
    url(r'^report/compare_unique$', report_views.compare_unique, name='compare_unique'),
    url(r'^report/filted_suspicious$', report_views.filted_suspicious, name='filted_suspicious'),
    # 一键功能
    url(r'^one_button$', views.one_button, name='one_button'),
    # 快照任务
    url(r'^add_task$', task_views.add_task, name='add_task'),
    url(r'^get_tasks$', task_views.get_tasks, name='get_tasks'),
    url(r'^edit_task$', task_views.edit_task, name='edit_task'),
    # 日志模块
    url(r'^real_time_log$', log_views.real_time_log, name='real_time_log'),

    # ======================================================================================
    # 后端接口
    # 用户相关
    url('^login_action$', user_views.login_action, name='login_action'),
    url('^logout$', user_views.logout, name='logout'),
    url('^register_action$', user_views.register_action, name='register_action'),
    url('^invite_code_action$', user_views.invite_code_action, name='invite_code_action'),
    url('^retrieve_email_action$', user_views.retrieve_email_action, name='retrieve_email_action'),
    url('^retrieve_action$', user_views.retrieve_action, name='retrieve_action'),
    # CURD
    url(r'^pubsocs$', db_views.pubsocs, name='pubsocs'),
    url(r'^pubsocs/(?P<id>[0-9]+)$', db_views.pubsocs, name='pubsocs'),
    url(r'^pubsocs_jqgrid$', db_views.pubsocs_jqgrid, name='pubsocs_jqgrid'),
    url(r'^prisocs$', db_views.prisocs, name='prisocs'),
    url(r'^prisocs/(?P<id>[0-9]+)$', db_views.pubsocs, name='prisocs'),
    url(r'^prisocs_jqgrid$', db_views.prisocs_jqgrid, name='prisocs_jqgrid'),
    url(r'^malicious_domains/(?P<id>[0-9]+)$', db_views.malicious_domains, name='malicious_domains'),
    url(r'^malicious_domains_jqgrid$', db_views.malicious_domians_jqgrid, name='malicious_domains_jqgrid'),
    url(r'^snapshots$', db_views.snapshots, name='snapshots'),
    url(r'^snapshots/(?P<id>[0-9]+)$', db_views.snapshots, name='snapshots'),
    url(r'^snapshots_jqgrid$', db_views.snapshots_jqgrid, name='snapshots_jqgrid'),
    url(r'^suspicious_records$', db_views.suspicious_records, name='suspicious_records'),
    url(r'^suspicious_records/(?P<id>[0-9]+)$', db_views.suspicious_records, name='suspicious_records'),
    url(r'^suspicious_records_jqgrid$', db_views.suspicious_records_jqgrid, name='suspicious_records_jqgrid'),
    url(r'^pocr_records$', db_views.pocr_records, name='pocr_records'),
    url(r'^pocr_records/(?P<id>[0-9]+)$', db_views.pocr_records, name='pocr_records'),
    url(r'^pocr_jqgrid$', db_views.pocr_jqgrid, name='pocr_jqgrid'),
    url(r'^dcbps/(?P<id>[0-9]+)$', db_views.dcbps, name='dcbps'),
    url(r'^dcbps_jqgrid$', db_views.dcbps_jqgrid, name='dcbps_jqgrid'),
    # 数据报表
    url('^redirect_records_datas$', report_views.redirect_records_datas, name="redirect_records_datas"),
    url('^compare_unique_datas$', report_views.compare_unique_datas, name="compare_unique_datas"),
    url('^filted_suspicious_datas$', report_views.filted_suspicious_datas, name="filted_suspicious_datas"),
    # 人工后处理
    url(r'^check_compare_unique$', report_views.check_compare_unique, name="check_compare_unique"),
    url(r'^check_suspicious$', report_views.check_suspicious, name="check_suspicious"),
    # ssh
    url('^beat_start$', views.beat_start, name="beat_start"),
    url('^beat_stop$', views.beat_stop, name="beat_stop"),
    url('^beat_restart$', views.beat_restart, name="beat_restart"),
    # 快照任务
    url(r'^tasks$', task_views.tasks, name="tasks"),
    # 日志模块
    url(r'^get_log_files$', log_views.get_log_files, name="get_log_files")
}


#!/usr/bin/env python
# encoding: utf-8
# """
# Console.py
#
# Created by mikepk on 2011-10-16.
# Copyright (c) 2011 Michael Kowalchik. All rights reserved.
# """
import code

import readline
import atexit
import os

import webob
from ..test import Client

class Console(code.InteractiveConsole):
    '''Pybald console including history buffer per project.'''
    def __init__(self, project_name, package_name=None, app=None,
                 additional_symbols=None):
        if additional_symbols is None:
            additional_symbols = {}

        self.project_name = project_name
        if not package_name:
            package_name = self.project_name

        # add the application to the console namespace
        console_symbols = {'app': app}

        # add webob Req/Resp objects to the console for convenience
        console_symbols.update({"Request": webob.Request,
                                "Response": webob.Response})
        console_symbols.update({"c": Client(app=app)})
        console_symbols.update(additional_symbols)

        histfile = os.path.expanduser("~/.pybald-{0}-console-history".format(
                                                                 project_name))

        code.InteractiveConsole.__init__(self, locals=console_symbols,
                                                          filename="<console>")
        self.init_history(histfile)

    def init_history(self, histfile):
        readline.parse_and_bind("tab: complete")
        if hasattr(readline, "read_history_file"):
            try:
                readline.read_history_file(histfile)
            except IOError:
                pass
            atexit.register(self.save_history, histfile)

    def save_history(self, histfile):
        readline.write_history_file(histfile)

    def run(self):
        # Fire up the console with the project, controllers, and models defined.
        self.interact('''Welcome to the pybald interactive console\n'''
                      ''' ** project: {0} **'''.format(self.project_name))


import pygame
import sys
import math
import random
import copy
import profile
pygame.init()
chess_number = 20  # 设置棋盘大小
LIVEDENSITY = 1 # 生命密度
TICK = 5 # 帧数
BG = (255, 255, 255)
# BG = (20, 20, 20)  # 背景色
LINECOLOR = (52, 53, 46)  # 网格色
LIFECOLOR = (31, 97, 189)  # 活细胞的颜色
black = (0, 0, 0)
CELL_LENGTH = int(600 / chess_number)  # 每个格子的像素大小
LINE_WIDTH = 2  # 线的宽度
START_POSX = 50
START_POSY = 50

gameDisplay = pygame.display.set_mode( (chess_number ,chess_number ) )

# 设置背景框大小
size = width, height = 2 * START_POSX + chess_number * CELL_LENGTH, 2 * START_POSY + chess_number * CELL_LENGTH
# 设置帧率，返回clock 类
clock = pygame.time.Clock()
screen = pygame.display.set_mode(size)
pygame.display.set_caption("Accelerator made")

# 画
def draw(livcell):
    '''
    进行一次绘画操作，可以理解为进行一帧操作所更新的画面
    param:操作前活细胞图
    return:操作后活细胞图
    '''
    for i in range(chess_number + 1):
        pygame.draw.line(screen, LINECOLOR, (START_POSX, START_POSY + i * CELL_LENGTH),
                         (START_POSX + chess_number * CELL_LENGTH, START_POSY + i * CELL_LENGTH), LINE_WIDTH)  # 横线
        pygame.draw.line(screen, LINECOLOR, (START_POSX + i * CELL_LENGTH, START_POSY),
                         (START_POSX + i * CELL_LENGTH, START_POSY + chess_number * CELL_LENGTH), LINE_WIDTH)  # 竖线#
    # 画活细胞
    livcell = rule(livcell)
    print('drawnew', livcell)
    return livcell

def drawcell(i, j, cellkind):
    '''
    画出一个具体的方块
    param:行，列，方块颜色种类
    '''
    pygame.draw.rect(screen, cellkind,
                     [START_POSX + CELL_LENGTH * j + (LINE_WIDTH - 1), START_POSY + CELL_LENGTH * i + (LINE_WIDTH - 1),
                      CELL_LENGTH - LINE_WIDTH, CELL_LENGTH - LINE_WIDTH], 0)  # 终点.  Rect(left,top,width,height)

def creatlife(density):
    '''
    在初始状态下创造生命
    param:所要求生成生命细胞的密度
    return:初始生命细胞的位置图
    '''
    livcell = [[0] * chess_number for i in range(chess_number)]
    for i in range(chess_number):
        for j in range(chess_number):
            pwall = random.random()
            if pwall < density:
                livcell[i][j] = 1
    return livcell

def neighborcell(pos):
    '''
    获得一个细胞周围的细胞位置，并且存入数组
    param:细胞的位置
    return:这个细胞所有的邻居细胞
    '''
    neighborList = []
    x = pos[0]
    y = pos[1]
    neighborList = [[x - 1, y - 1], [x - 1, y], [x - 1, y + 1], [x, y - 1], [x, y + 1], [x + 1, y - 1], [x + 1, y],
                    [x + 1, y + 1]]

    realnList = copy.deepcopy(neighborList)
    for i in neighborList:
        if i[0] < 0 or i[0] > chess_number - 1 or i[1] < 0 or i[1] > chess_number - 1:
            realnList.remove(i)

    return realnList

def rule(livcell):
    '''
    制定生命游戏的游戏规则
    param:一次操作前所有活着细胞的位置
    return:一次操作后所有活着细胞的位置
    '''
    newlivcell = copy.deepcopy(livcell)
    print('livcell', livcell)
    for i in range(chess_number):
        for j in range(chess_number):
            if livcell[i][j] == 1:
                drawcell(i, j, LIFECOLOR)
                alive = 0
                for cell in neighborcell([i, j]):
                    if livcell[cell[0]][cell[1]] == 1:
                        alive += 1
                if alive == 0 or alive == 1:
                    newlivcell[i][j] = 0
                elif alive == 2 or alive == 3:
                    newlivcell[i][j] = 1
                else:
                    newlivcell[i][j] = 0
            else:
                alive = 0
                for cell in neighborcell([i, j]):
                    if livcell[cell[0]][cell[1]] == 1:
                        alive += 1
                if alive == 3:
                    newlivcell[i][j] = 1
    return newlivcell

def text_objects(text, font):
    textSurface = font.render(text, True, black)
    return textSurface, textSurface.get_rect()

def main():
    livcell = creatlife(LIVEDENSITY)
    while True:
        for event in pygame.event.get():
            # 查找关闭窗口事件
            if event.type == pygame.QUIT:
                sys.exit()
        # 填充背景色

        screen.fill(BG)
        livcell = draw(livcell)
        # 刷新图s
        pygame.display.flip()
        clock.tick(TICK)

if __name__ == "__main__":
    profile.run("main()")


#! /usr/bin/env python3
# -*- coding: utf-8 -*-
'''
This crawler is used to crawl a
web site and change it to pdf
'''

__author__ = 'ly'

from urllib import request
from urllib import parse
from urllib import error
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import pdfkit
import time, os, re, logging

class Crawler(object):
    '''
    爬虫基类，所有爬虫都应该继承此类
    '''
    name = None

    def __init__(self, name, start_url):
        '''
        初始化
        ：param name: 保存PDF文件名，不需要后缀名
        : param start_url: 爬虫入口url
        '''
        self.name = name
        self.start_url = start_url
	# 爬取网站的域名，但是这里没有使用这个
        self.domain = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(self.start_url))

    def crawl(self, url):
        '''
        return url
        '''
        return request.urlopen(url)

    def parse_menu(self, response):
        '''
        解析目录结构
        '''
        raise NotImplementedError

    def parse_body(self, response):
        '''
        解析正文
        '''
        raise NotImplementedError
    def parse_css(self, response):
        '''
        解析css(注： 可以有多个css文件，但是所有网页的css是一样的)
        '''
        return None

    def run(self):
        '''
        run method
        '''
        start = time.time()
        css_list = self.parse_css(self.crawl(self.start_url))
        menu_list = self.parse_menu(self.crawl(self.start_url))
        if not menu_list:
            logging.error('在起始url没有找到目录，程序返回', exc_info=True)
            return None
        # 对html、css等文件的出来
        # try except finally 方法
        html_template = '''
        <!DOCTYPE html>

            <meta charset="UTF-8">
        '''
        if css_list:
            for css in css_list:
                html_template = html_template + '\n <link rel="stylesheet" href="{}">'.format(css)
            html_template = html_template + '\n\n'
        with open('final.html', 'wb') as f_html:
            f_html.write(html_template.encode('utf-8'))
            for url in menu_list:
                cur_html = self.parse_body(self.crawl(url))
                print('抓取 "{}" 网页完成， 正在写入本地文件'.format(cur_html.h1.get_text()))
                f_html.write(cur_html.encode('utf-8'))
            f_html.write('\n\n'.encode('utf-8'))

        print('translate to "{}.pdf"!!'.format(self.name))
        try:
            pdfkit.from_file('final.html', self.name + ".pdf")
            # 调试的时候保留了html源文件，实际使用的时候可以删掉
            #os.remove('final.html')
        except Exception as e:
            logging.error('转化为pdf失败！\n', exc_info=True)
        total_time = time.time() - start
        print(u'总计耗时： %f 秒' % total_time)

class WebCrawler(Crawler):
    '''
    廖雪峰python3教程
    '''

    def parse_menu(self, response):
        '''
        解析目录结构
        '''
        bs_obj = BeautifulSoup(response, 'lxml')
        nav_div = bs_obj.find('div', class_='x-sidebar-left-content')
        link_list = nav_div.findAll('a', {'href' : re.compile('/wiki/')})
        for alist in link_list:
            src = alist['href']
            if not src.startswith('http'):
                src = urljoin(self.start_url, src)
            yield src

    def parse_css(self, response):
        '''
        解析css文件
        '''
        bs_obj = BeautifulSoup(response, 'lxml')
        css_label = bs_obj.head.findAll('link', {'href' : re.compile(r'.*?\.css')})
        for alabel in css_label:
            yield urljoin(self.start_url, alabel['href'])

    def parse_body(self, response):
        '''
        解析正文
        '''
        try:
            bs_obj = BeautifulSoup(response, 'lxml')
            title = bs_obj.h4.get_text()
            main_page = bs_obj.find('div', class_='x-wiki-content')

            new_center = bs_obj.new_tag('center')
            new_title = bs_obj.new_tag('h1')
            new_title.string = title
            new_center.insert(1, new_title)
            main_page.insert(1, new_center)

            for img in main_page.findAll('img', {'src' : re.compile('^(?!http)')}):
                img['src'] = urljoin(self.start_url, img['src'])

            return main_page
        except Exception as e:
            logging.error('解析错误!\n', exc_info=True)

if __name__ == '__main__':
    start_url = 'http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000'
    crawler = WebCrawler('python3', start_url)
    crawler.run()


from django.db import models
from django.utils.translation import ugettext_lazy as _

# Create your models here.
class SlideShow(models.Model):
    title = models.CharField(_('Title'), max_length=150)
    subTitle = models.CharField(_('SubTitle'), max_length=150)
    image = models.ImageField(_('Background'), upload_to='site/slideShow')
    action_text = models.CharField(_('action text'), max_length=150, default='shop now')
    action_url = models.URLField(_('action url'), default='http://127.0.0.1:8000')

    def __str__(self):
        return self.title


from flask import request, render_template
from app import app, db
from app.database import Message

@app.post("/messages")
def create_message():
    msg_data = request.json
    title = msg_data.get("title")
    body = msg_data.get("body")
    if not title or not body:
        return "
Invalid syntax
", 400
    message = Message(title=title, body=body)
    db.session.add(message)
    db.session.commit
    return "
Created
", 201

@app.get("/messages")
def read_all_messages():
    messages = Message.query.all()
    return render_template("message_list.html", message_list=messages)

@app.route("/messages/<int:pk>")
def read_message_by_id(pk):
    message=Message.query.filter_by(id=pk).first()
    return render_template("message_detail.html",message=message )

@app.get("/greeting/<name>")
def greet(name):
    return render_template("home.html", username=name)


import os, sys
import json
import requests
from bs4 import BeautifulSoup
import urllib
import time
import selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from datetime import datetime
import pymongo
from pymongo import MongoClient
import geocoder
# 0 : test
def test():
    g = geocoder.google('Mountain View, CA')
    print(g.lat)

#test()
# 1: crawling from otodom and save in the otodom_body.txt
def crawling_otodom():
    try:

        url = "https://otodom.pl/sprzedaz/mieszkanie/warszawa/?search%5Bdist%5D=0"

        # FireFox 를 이용하여 크롤링 시작 #

        driver = webdriver.Firefox()

        driver.implicitly_wait(30)
        driver.get(url)
        for index in range(0,100): # 100 page * 25 posts = 2,500 posts crawling
            try:
                for i in range(4,34):
                    time.sleep(1)  # 10초간 데이터 받기를 기다립니다
                    depart_element = driver.find_elements_by_xpath("//article")
                    driver.execute_script("window.scrollTo(0, " + str(i*10) + ");")
                    depart_element[i].click()

                    elem = driver.find_element_by_xpath("//body")
                    source_code = elem.get_attribute("outerHTML")
                    #print(source_code)
                    f = open("./templates/log/response/otodom_body.txt", 'w')
                    enc_response = urllib.parse.unquote(source_code)
                    f.write(enc_response)
                    f.close()
                    data_processing_otodom()
                    driver.back()
            except Exception as E:
                print("otom_crawling fail internal iteration : " + str(E))
                pass
            depart_element = driver.find_elements_by_xpath("//a[@data-dir='next']")
            depart_element[0].click()
            time.sleep(5)
        driver.quit()
        print("crwaling finished")
        print(datetime.now().strftime("%H:%M:%S"))
        # 크롤링 종료
    except Exception as E:
        print("otom_crawling fail overall : " + str(E))
        print(datetime.now().strftime("%H:%M:%S"))
        pass

#2 : load the otodom_body.txt and save in the mongoDB

def data_processing_otodom():
    try:
        url = "./templates/log/response/otodom_body.txt"
        page = open(url)
        soup = BeautifulSoup(page.read(), "lxml")

        otodom_json_catched = str(soup).split("GPT.targeting = ")[1].split(";")[0]
        print(otodom_json_catched)
        items = json.loads(otodom_json_catched)
        items['crawling_website'] = "otodom"

        address_catched = soup.find_all("p", class_="address-links")[0]
        address = address_catched.find_all("a")[-2].text.strip()
        items['Address'] = address
        items['Address_nospace'] = address.replace(" ","")
        g = geocoder.google(items['Address'])
        items['lat'] = g.lat
        items['lng'] = g.lng

        thumbnail_catched = soup.find_all("a", class_="gallery-box-image-item atlas-slider-item")[0]
        thumbnail = thumbnail_catched.find_all("img")[0]["src"]
        items['Thumbnail'] = thumbnail

        postname_catched = soup.find_all("header", class_="col-md-offer-content")[0]
        postname = postname_catched.find_all("h1")[0].text.replace(">","").strip()
        items['Postname'] = postname

        crwaling_date = datetime.now().strftime('%Y/%m/%d')
        items['Crawling_date'] = crwaling_date

        client = MongoClient()
        client = MongoClient('localhost', 27017)
        db = client["Crawling_Info_For_Flats_In_Poland"]
        collection = db["test_collection"]
        posts = db.posts
        print("insert to Mongo DB")
        post_id = posts.insert(items)

    except Exception as E:
        print("otodom_crawling fail : " + str(E))
        pass

crawling_otodom()
#data_processing_otodom()


import os
import json
from lib.model.json import NewJson
from lib.model.case import Case
from lib.model.config import ConfigManager

config = ConfigManager()
# Initial Quality check of new json
unprocessed_jsons = os.path.join(config.aws['download_location'], 'cases')
case_id_list = [
    # INSERT CASE IDS HERE
]

still_empty = [
    # INSERT CASE IDS FOR SECOND PROCESSING
]

case_id_list = still_empty

json_files = [os.path.join(unprocessed_jsons, x)
              for x in os.listdir(unprocessed_jsons)
              if os.path.splitext(x)[1] == '.json'
              and int(os.path.splitext(x)[0]) in case_id_list]

corrected = config.preprocess['corrected_location']
new_json_objs = [NewJson.from_file(f, corrected) for f in json_files]
raw_dict = {}
for j in new_json_objs:
    print(j.get_case_id())
    print(j.get_js()['genomic_entries'])

# new_cases = [Case(j) for j in new_json_objs if j.check()[0]]

# create a list of case_id to gene_name
# case_json = [{c.case_id: j.entry_id} for c in new_cases for j in c.hgvs_models]
# print(case_json)
# case_json = [{c.case_id: j.get_json()} for c in new_cases for j in c.hgvs_models]
# json.dump(case_json, open('case_json_entries.json', 'w'), indent=4)
# gene_dict = {}
# case_ids = []
# case_ids_added = []
# for c in new_cases:
#     for j in c.hgvs_models:
#         case_ids.append(c.case_id)
#         if j.gene['gene_symbol']:
#             gene_dict[c.case_id] = j.gene['gene_symbol']
#             case_ids_added.append(c.case_id)
# case_ids = set(case_ids)
# case_ids_added = set(case_ids_added)
# case_diff = case_ids - case_ids_added
# print(case_diff)
# json.dump(gene_dict, open('gene_dict.json', 'w'), indent=4)
# print(gene_dict)

# case_gene_name = [{c.case_id: j.gene['gene_symbol']}
#                   for c in new_cases for j in c.hgvs_models]
#
# empty_cases = [ c for c in case_gene_name if list(c.values())[0] == ""]
# print(empty_cases)
# # json.dump(case_gene_name, open('case_gene_name.json', 'w'))


# -*- coding: utf-8 -*-
"""
    analytics.views

    Views for analytics

    :copyright: (c) 2013 by Openlabs Technologies & Consulting (P) Limited
    :license: BSD, see LICENSE for more details.
"""
import json
from django.template import Context
from django.http import HttpResponse
from django.db import connection
from django.db.models import Sum, Count, Max
from django.template import RequestContext
from django.shortcuts import render_to_response

from admin.models import SubSector, Institution
from analytics.models import *

def get_latest_year(type):
    """
        Return latest year for analytics
    """
    Base = DemandData if type == 'demand' else SupplyBase
    return Base.objects.all().aggregate(year=Max('year'))['year']

def home(request):
    "Home page. What to show?"
    demand_years = sorted([
        k['year'] for k in DemandData.objects.values('year').distinct()
    ], reverse=True)
    supply_years = sorted([
        k['year'] for k in SupplyBase.objects.values('year').distinct()
    ], reverse=True)

    c = Context({
        'analytics_demand_years': demand_years,
        'analytics_supply_years': supply_years
    })
    return render_to_response('analytics.html', c,
            context_instance=RequestContext(request))

def demand(request, year=None):
    "Demand main page"
    year = int(year) if year else get_latest_year('demand')
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/demand.html', c,
            context_instance=RequestContext(request))

def supply(request, year=None):
    "Supply main page"
    year = int(year) if year else get_latest_year('supply')
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/supply.html', c,
            context_instance=RequestContext(request))

def skillgaps(request, year=None):
    "Skillgaps main page"
    year = int(year) if year else get_latest_year('demand')
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/skillgaps.html', c,
            context_instance=RequestContext(request))

def demand_3(request, year):
    "Demand #3 page"
    year = int(year)
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/demand-3.html', c,
            context_instance=RequestContext(request))

def supply_2(request, year):
    "Supply #2 page"
    year = int(year)
    c = Context({
        'analytics_year': year,
    })
    return render_to_response('analytics/supply-2.html', c,
            context_instance=RequestContext(request))

def demand_by_state(request, year):
    "Returns a list of states and their demand"
    year = int(year)

    cursor = connection.cursor()
    cursor.execute('''
        SELECT DemandT.year AS year,
           StateT.id AS state_id,
           StateT.name AS state,
           sum(DemandT.demand) AS demand
        FROM analytics_demanddata AS DemandT,
             analytics_state AS StateT,
             analytics_city AS CityT
        WHERE year = %s AND CityT.id = DemandT.city_id
                        AND CityT.state_id = StateT.id
        GROUP BY state;
    ''', (year,))

    states = {}
    demand_data = {}
    for _, state_id, state, demand in cursor.fetchall():
        demand = int(demand)
        demand_data[state] = demand
        states[state] = state_id

    return HttpResponse(json.dumps({
        'year': year,
        'states': states,
        'data': demand_data,
    }), content_type='text/json')

def demand_in_state(request, year, state_id):
    "Returns complete JSON data to drilldown in given state"
    year = int(year)
    state_id = int(state_id)
    state_name = State.objects.get(pk=state_id).name

    cursor = connection.cursor()
    cursor.execute('''
        SELECT
            DemandT.city_id AS city_id,
            CityT.name AS city,
            sum(demand) AS demand
        FROM
            analytics_demanddata AS DemandT,
            analytics_city AS CityT,
            analytics_state AS StateT
        WHERE
            DemandT.year = %s AND
            StateT.id = %s AND
            CityT.state_id = StateT.id AND
            DemandT.city_id = CityT.id
        GROUP BY city_id;
    ''', (year, state_id))

    cities = []
    data = []

    for city_id, city, demand in cursor.fetchall():
        cities.append(city)
        data.append({
            'y': int(demand),
            'drilldown': _demand_subsector(year, int(city_id))
        })

    json_data = {
        'name': "Demand in %s (%d)" % (state_name, year),
        'categories': cities,
        'data': data,
    }

    return HttpResponse(json.dumps({
        'year': year,
        'state': state_name,
        'data': json_data,
    }), content_type='text/json')

def _demand_subsector(year, city_id):
    "Return drilldown JSON data for given subsector"

    city = City.objects.get(pk=city_id)
    city_name = city.name
    state_name = city.state.name

    cursor = connection.cursor()
    cursor.execute('''
        SELECT
            SubSectorT.id AS subsector_id,
            SubSectorT.name AS subsector,
            sum(demand) AS demand
        FROM
            analytics_demanddata AS DemandT,
            admin_subsector AS SubSectorT,
            admin_occupation AS OccupationT
        WHERE
            DemandT.year = %s AND
            DemandT.city_id = %s AND
            OccupationT.id = DemandT.occupation_id AND
            SubSectorT.id = OccupationT.sub_sector_id
        GROUP BY subsector_id;
    ''', (year, city_id))

    subsectors = []
    data = []

    for subsector_id, subsector, demand in cursor.fetchall():
        subsectors.append(subsector)
        data.append({
            'y': int(demand),
            'drilldown': _demand_occupation(year, city_id, int(subsector_id))
        })

    json_data = {
        'name': "Demand in %s, %s (%d) by Sub Sector" %
                        (city_name, state_name, year),
        'categories': subsectors,
        'data': data,
    }
    return json_data

def _demand_occupation(year, city_id, subsector_id):
    "Return drilldown JSON data for given subsector"

    city = City.objects.get(pk=city_id)
    city_name = city.name
    state_name = city.state.name
    subsector_name = SubSector.objects.get(pk=subsector_id).name

    cursor = connection.cursor()
    cursor.execute('''
        SELECT
            DemandT.occupation_id AS occupation_id,
            OccupationT.name AS occupation,
            sum(demand) AS demand
        FROM
            analytics_demanddata AS DemandT,
            admin_subsector AS SubSectorT,
            admin_occupation AS OccupationT
        WHERE
            DemandT.year = %s AND
            DemandT.city_id = %s AND
            SubSectorT.id = %s AND
            OccupationT.id = DemandT.occupation_id
        GROUP BY occupation_id;
    ''', (year, city_id, subsector_id))

    occupations = []
    data = []

    for occupation_id, occupation, demand in cursor.fetchall():
        occupations.append(occupation)
        data.append({
            'y': int(demand)
        })

    json_data = {
        'name': "Demand in %s, %s (%d) by Occupations for %s" %
                        (city_name, state_name, year, subsector_name),
        'categories': occupations,
        'data': data,
    }
    return json_data

def supply_by_state(request, year):
    "Returns a list of states and their supply"
    year = int(year)

    cursor = connection.cursor()
    cursor.execute('''
        SELECT SupplyT.year AS year,
            StateT.id AS state_id,
            StateT.name AS state,
            sum(SupplyT.supply) AS supply
        FROM analytics_supplybase AS SupplyT,
            analytics_state AS StateT,
            analytics_city AS CityT
        WHERE year = %s AND CityT.id = SupplyT.city_id
                        AND CityT.state_id = StateT.id
        GROUP BY state;
    ''', (year,))

    states = {}
    supply_data = {}
    for _, state_id, state, supply in cursor.fetchall():
        supply = int(supply)
        supply_data[state] = supply
        states[state] = state_id

    return HttpResponse(json.dumps({
        'year': year,
        'states': states,
        'data': supply_data,
    }), content_type='text/json')

def supply_in_state(request, year, state_id):
    "Returns complete JSON data to drilldown in given state"
    year = int(year)
    state_id = int(state_id)
    state_name = State.objects.get(pk=state_id).name

    cursor = connection.cursor()
    cursor.execute('''
        SELECT
            SupplyT.city_id AS city_id,
            CityT.name AS city,
            sum(supply) AS supply
        FROM
            analytics_supplybase AS SupplyT,
            analytics_city AS CityT,
            analytics_state AS StateT
        WHERE
            SupplyT.year = %s AND
            StateT.id = %s AND
            CityT.state_id = StateT.id AND
            SupplyT.city_id = CityT.id
        GROUP BY city_id;
    ''', (year, state_id))

    cities = []
    data = []

    for city_id, city, supply in cursor.fetchall():
        cities.append(city)
        data.append({
            'y': int(supply),
            'drilldown': _supply_subsector(year, int(city_id))
        })

    json_data = {
        'name': "Supply in %s (%d)" % (state_name, year),
        'categories': cities,
        'data': data,
    }

    return HttpResponse(json.dumps({
        'year': year,
        'state': state_name,
        'data': json_data,
    }), content_type='text/json')

def _supply_subsector(year, city_id):
    "Return drilldown JSON data for given subsector"

    city = City.objects.get(pk=city_id)
    city_name = city.name
    state_name = city.state.name

    cursor = connection.cursor()
    cursor.execute('''
        SELECT
            SubSectorT.id AS subsector_id,
            SubSectorT.name AS subsector,
            sum(supply) AS supply
        FROM
            analytics_supplybase AS SupplyT,
            admin_subsector AS SubSectorT,
            admin_occupation AS OccupationT
        WHERE
            SupplyT.year = %s AND
            SupplyT.city_id = %s AND
            OccupationT.id = SupplyT.occupation_id AND
            SubSectorT.id = OccupationT.sub_sector_id
        GROUP BY subsector_id;
    ''', (year, city_id))

    subsectors = []
    data = []

    for subsector_id, subsector, supply in cursor.fetchall():
        subsectors.append(subsector)
        data.append({
            'y': int(supply),
            'drilldown': _supply_occupation(year, city_id, int(subsector_id))
        })

    json_data = {
        'name': "Supply in %s, %s (%d) by Sub Sector" %
                        (city_name, state_name, year),
        'categories': subsectors,
        'data': data,
    }
    return json_data

def _supply_occupation(year, city_id, subsector_id):
    "Return drilldown JSON data for given subsector"

    city = City.objects.get(pk=city_id)
    city_name = city.name
    state_name = city.state.name
    subsector_name = SubSector.objects.get(pk=subsector_id).name

    cursor = connection.cursor()
    cursor.execute('''
        SELECT
            SupplyT.occupation_id AS occupation_id,
            OccupationT.name AS occupation,
            sum(supply) AS supply
        FROM
            analytics_supplybase AS SupplyT,
            admin_subsector AS SubSectorT,
            admin_occupation AS OccupationT
        WHERE
            SupplyT.year = %s AND
            SupplyT.city_id = %s AND
            SubSectorT.id = %s AND
            OccupationT.id = SupplyT.occupation_id
        GROUP BY occupation_id;
    ''', (year, city_id, subsector_id))

    occupations = []
    data = []

    for occupation_id, occupation, supply in cursor.fetchall():
        occupations.append(occupation)
        data.append({
            'y': int(supply)
        })

    json_data = {
        'name': "Supply in %s, %s (%d) by Occupations for %s" %
                        (city_name, state_name, year, subsector_name),
        'categories': occupations,
        'data': data,
    }
    return json_data

# Analytics 1

def revenue_company(request, year):
    "Return JSON string with revenue for nasscom and non-nasscom companies"
    queryset = CompanyYearData.objects.filter(year=year)
    nasscom_members = queryset.exclude(
        company__nasscom_membership_number__exact='N/A'
    ).aggregate(count=Count('revenue'), revenue=Sum('revenue'))
    nasscom_members['revenue'] = nasscom_members['revenue'] or 0

    non_nasscom_members = queryset.filter(
        company__nasscom_membership_number__exact='N/A'
    ).aggregate(count=Count('revenue'), revenue=Sum('revenue'))
    non_nasscom_members['revenue'] = non_nasscom_members['revenue'] or 0

    total = nasscom_members['revenue'] + non_nasscom_members['revenue']
    percent = lambda x: (x * 100.0) / (total or 1)

    return HttpResponse(json.dumps({
        'nasscom_members_num': nasscom_members['count'],
        'nasscom_members_revenue': percent(nasscom_members['revenue']),
        'non_nasscom_members_num': non_nasscom_members['count'],
        'non_nasscom_members_revenue': percent(non_nasscom_members['revenue']),
    }), content_type='text/json')

def revenue_company_type(request, year):
    "Return JSON string with revenue grouped by company type"
    result = CompanyYearData.objects.filter(
        year=year,
    ).values('company__company_type') \
        .annotate(revenue=Sum('revenue'))

    return_result = []
    total = CompanyYearData.objects.filter(year=year). \
            aggregate(sum=Sum('revenue'))['sum']
    percent = lambda x: (x * 100.0) / (total or 1)

    for item in result:
        return_result.append((item['company__company_type'],
            percent(item['revenue']), ))

    return HttpResponse(json.dumps(return_result), content_type='text/json')

def _demanddata_contribution(year, field, threshold_company_collect):
    "Return cumulative sum by grouping field"
    points = [4, 10, 20, 50]
    cumulative_sums = []
    companies = []
    field_cumulative = 0
    resultset = DemandData.objects.filter(year=year) \
            .values('company', 'company__name') \
            .annotate(field=Sum(field)).order_by('-%s' % field)
    field_total = DemandData.objects.filter(year=year). \
            aggregate(total=Sum(field))['total']
    count = resultset.count()
    for index, result in enumerate(resultset, 1):
        field_cumulative += result['field']
        percent = (field_cumulative * 100.0) / (field_total or 1)
        if (index in points) or index == count:
            cumulative_sums.append(
                (index, percent)
            )
        if percent >= threshold_company_collect:
            companies.append(result['company__name'])
    return {
        'cumulative_sums': cumulative_sums,
        'threshold': threshold_company_collect,
        'companies': companies
    }

def headcount_contribution(request, year):
    "Return cumulative headcount contribution top companies"
    return HttpResponse(json.dumps(
        _demanddata_contribution(year, 'headcount', 80)
    ))

def hiring_contribution(request, year):
    "Return cumulative headcount contribution top companies"
    return HttpResponse(json.dumps(
        _demanddata_contribution(year, 'demand', 80)
    ))

def hiring_subsector_trend(request):
    "Return hiring for year by sub sector"
    resultset = DemandData.objects.values(
        'occupation__sub_sector',
        'occupation__sub_sector__name',
        'year',
    ).annotate(hiring=Sum('demand')).order_by('year')
    data = {}
    years = resultset.values('year').distinct()
    years = map(lambda r: r['year'], years)

    for result in resultset:
        year = result['year']
        sub_sector = result['occupation__sub_sector__name']
        hiring = result['hiring']

        if sub_sector not in data:
            data[sub_sector] = []
        data[sub_sector].append((year, hiring,))

    trend_data = []
    for sub_sector, points in data.items():
        trend_data.append({
            'name': sub_sector,
            'data': points
        })

    return HttpResponse(json.dumps({
        'series': trend_data,
        'years': years,
    }))

def demand_1(request, year):
    "Analytics 1 page"
    year = int(year)
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/demand-1.html', c,
            context_instance=RequestContext(request))

# Analytics 1

def talent_saturation(request, year):
    "JSON data for talent saturation"
    record = TalentSaturation.objects.get(year=year)
    return HttpResponse(json.dumps(record.series()))

def demand_2(request, year):
    "Analytics 1 page"
    year = int(year)
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/demand-2.html', c,
            context_instance=RequestContext(request))

# Analytics 4

def demand_supply_region(request, year):
    year = int(year)
    demand_data = []
    supply_data = []

    for region_key, region_value in REGION_CHOICES:
        demand = DemandData.objects.filter(
            year=year,
            city__state__region=region_key
        ).aggregate(demand=Sum('demand'))
        demand = demand['demand'] if demand else None
        demand_data.append([region_value, demand])

        supply = SupplyBase.objects.filter(
            year=year,
            city__state__region=region_key
        ).aggregate(supply=Sum('supply'))
        supply = supply['supply'] if supply else None
        supply_data.append([region_value, supply])

    return HttpResponse(json.dumps({
        'demand': demand_data,
        'supply': supply_data,
    }), content_type='text/json')

def it_spend(request, year):
    """
    Return JSON data for IT spend by world and indian revenue for
    each subsector
    """
    year = int(year)
    resultset = ITSpend.objects.filter(year=year)
    world_spend = []
    india_revenue = []

    for result in resultset:
        world_spend.append({
            'name': result.sub_sector.name,
            'data': [result.world_spend]
        })
        india_revenue.append({
            'name': result.sub_sector.name,
            'data': [result.india_revenue]
        })

    return HttpResponse(json.dumps({
        'world_spend': world_spend,
        'india_revenue': india_revenue,
    }), content_type='text/json')

def revenue_subsector_trend(request):
    """
    JSON data containing revenue each year split by subsector
    """
    years = sorted([
        k['year'] for k in RevenueSubsector.objects.values('year').distinct()
    ])
    data = {}
    return_data = []

    for year in years:
        resultset = RevenueSubsector.objects.filter(year=year) \
                .order_by('sub_sector')
        for result in resultset:
            sub_sector = result.sub_sector.name
            if sub_sector not in data:
                data[sub_sector] = []
            data[sub_sector].append(result.revenue)

    for key, value in data.items():
        return_data.append({
            'name': key,
            'data': value,
        })
    return HttpResponse(json.dumps({
        'series': return_data,
        'years': years,
    }), content_type='text/json')

def revenue_occupation(request, year):
    """
    JSON data containing revenue by occupation in year=year and year = year + 7
    """
    year = int(year)
    resultset = RevenueOccupation.objects.filter(year=year)
    data = []
    data7 = []  # data after 7 years
    occupations = []

    total, total7 = 0, 0
    for result in resultset:
        occupations.append(result.occupation.name)
        data.append(result.revenue)
        data7.append(result.revenue_after_7year)
        total += result.revenue
        total7 += result.revenue_after_7year

    data = map(lambda x: (x * 100.0) / total, data)
    data7 = map(lambda x: (x * 100.0) / total7, data7)

    return HttpResponse(json.dumps({
        'occupations': occupations,
        'series': [
            {'name': year, 'data': data},
            {'name': year + 7, 'data': data7},
        ]
    }), content_type='text/json')

def demand_4(request, year):
    "Analytics 4 page"
    year = int(year)
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/demand-4.html', c,
            context_instance=RequestContext(request))

# Analytics 5

def diversity_ratio_level(request, year):
    "Diversity ratio by level JSON data"
    year = int(year)
    male = []
    female = []
    resultset = DiversityRatioLevel.objects.filter(year=year)
    for result in resultset:
        male.extend([
            result.male_entry,
            result.male_middle,
            result.male_leadership,
        ])
        female.extend([
            result.female_entry,
            result.female_middle,
            result.female_leadership,
        ])

    return HttpResponse(json.dumps({
        'male': male,
        'female': female,
        'categories': [
            'Entry Level (0-2) years',
            'Middle Level (2-10) years',
            'Leadership Level (>10) years',
        ]
    }), content_type='text/json')

def diversity_ratio_subsector(request, year):
    "Diversity ratio by roles JSON data"
    year = int(year)
    male = []
    female = []
    categories = []
    resultset = DiversityRatioSubsector.objects.filter(year=year)
    for result in resultset:
        male.append([result.subsector.name, result.male])
        female.append([result.subsector.name, result.female])
        categories.append(result.subsector.name)

    return HttpResponse(json.dumps({
        'male': male,
        'female': female,
        'categories': categories
    }), content_type='text/json')

def total_revenue_series(request, year):
    "Total revenue series + projected revenues"
    year = int(year)
    revenuetotal = RevenueTotal.objects.get(year=year)
    return HttpResponse(
        json.dumps(revenuetotal.growth_series),
        content_type='text/json'
    )

def demand_5(request, year):
    "Diversity ratio page"
    year = int(year)
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/demand-5.html', c,
            context_instance=RequestContext(request))

# Analytics 6

def demand_6(request, year):
    "Analytics 6 page"
    year = int(year)
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/demand-6.html', c,
            context_instance=RequestContext(request))

# Analytics 7

def demand_7(request, year):
    "Analytics 7 page"
    year = int(year)
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/demand-7.html', c,
            context_instance=RequestContext(request))

###### Supply Analytics #######

# Analytics 1

def university_in_states(request):
    "List number of universities in each state"
    result = Institution.objects.filter(
                is_university=True,
            ).values(
                'city__state__name'
            ).annotate(
                total=Count('id')
            ).order_by('-total').all()
    top_states = [k['city__state__name'] for k in result]

    states_data = {}
    for univ_type_key, univ_type in Institution.UNIVERSITY_CHOICES:
        results = Institution.objects.filter(
                    is_university=True,
                    university_type=univ_type_key
                ).values(
                    'city__state__name'
                ).annotate(
                    total=Count('id')
                )
        if not result:
            continue

        for result in results:
            state = result['city__state__name']
            if state not in states_data:
                states_data[state] = []
                states_data[state].append(0)

            states_data[state].append((univ_type_key, result['total']))
            states_data[state][0] += result['total']

    return HttpResponse(json.dumps({
        'states_data': states_data,
        'top_states': top_states,
        'university_type': dict(Institution.UNIVERSITY_CHOICES),
    }), content_type='text/json')

def supply_1(request):
    "Universities by state and top states"
    return render_to_response('analytics/supply-1.html', Context({}),
            context_instance=RequestContext(request))

# Analytics 3

def gender_diversity_data(request, year):
    """
    Return json data for gender diversity
    """
    resultset = GenderDiversity.objects.filter(
        year=year
    )
    male = []
    female = []
    categories = []

    for result in resultset:
        categories.append(result.category)
        male.append((result.category, result.male, ))
        female.append((result.category, 100 - result.male, ))

    return HttpResponse(json.dumps({
        'series': [
            {'name': 'Male', 'data': male},
            {'name': 'Female', 'data': female}
        ],
        'categories': categories,
        'year': year,
    }), content_type='text/json')

def supply_3(request, year):
    """
    Gender diversity page
    """
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/supply-3.html', c,
            context_instance=RequestContext(request))

# Analytics 4

def supply_4(request, year):
    "Where does talent move to?"
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/supply-4.html', c,
            context_instance=RequestContext(request))

# Analytics 5

def supply_split_stream_trend(request):
    """
    Return json data for supply split by stream
    """
    years = sorted([
        k['year'] for k in SupplyBase.objects.values('year').distinct()
    ])

    sectors_queryset = SupplyBase.objects.values(
        'occupation__sub_sector__sector__name'
    ).distinct().order_by('occupation__sub_sector__sector__name')
    sectors = sorted([
        k['occupation__sub_sector__sector__name'] for k in sectors_queryset
    ])
    sectors_set = set(sectors)

    data = {}
    for sector in sectors:
        data[sector] = []

    for year in years:
        resultset = SupplyBase.objects.filter(year=year).values(
            'occupation__sub_sector__sector',
            'occupation__sub_sector__sector__name',
        ).annotate(supply=Sum('supply')) \
                .order_by('occupation__sub_sector__sector')

        seen_sectors = set()
        for result in resultset:
            sector = result['occupation__sub_sector__sector__name']
            seen_sectors.add(sector)
            data[sector].append(result['supply'])

        unseen_sectors = sectors_set - seen_sectors
        for sector in unseen_sectors:
            data[sector].append(None)

    prepared_data = []
    for sector, values in data.items():
        prepared_data.append({
            'name': sector,
            'data': values,
        })

    return HttpResponse(json.dumps({
        'data': prepared_data,
        'categories': years
    }), content_type='text/json')

def supply_5(request, year):
    """
    Supply analytics 5
    """
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/supply-5.html', c,
            context_instance=RequestContext(request))

# Analytics 6

def supply_6(request, year):
    """
        What is the relationship between number of student supply and
        diversity ratio?
    """
    c = Context({
        'analytics_year': year
    })
    return render_to_response('analytics/supply-6.html', c,
            context_instance=RequestContext(request))

#### Skill Gaps ####

def skillgaps_slides(request, num):
    """
    Skill Gaps
    """
    num = int(num)

    if num == 1:
        context = Context({
            'analytics_title': 'How do skillgaps vary?',
            'analytics_slides': range(158, 174) + range(175, 180) +
                                range(26, 29),
        })
    elif num == 2:
        context = Context({
            'analytics_title': 'Which skills have highest gaps as viewed by '
                               'by industry?',
            'analytics_slides': [41, 160, 161, 164, 165, 168, 169, 172, 173]
        })
    elif num == 3:
        context = Context({
            'analytics_title': 'Amongst the identified hot skills which skills '
                               'have the highest positive/negative gaps?',
            'analytics_slides': range(158, 174),
        })
    elif num == 4:
        context = Context({
            'analytics_title': 'What are the skill gaps for more employable '
                               'skills?',
            'analytics_slides': range(158, 174),
        })

    return render_to_response('analytics/skillgaps-1.html', context,
            context_instance=RequestContext(request))


from sklearn.decomposition import PCA
import pandas as pd

def assert_None(args,
                raise_error=True):
    '''Make sure args are not None
    '''
    if any(x is None for x in args):
        num_none = sum(x is None for x in args)
        if raise_error:
            string='Nones detected : {}'.format(str(num_none))
            raise ValueError(string)
        else:
            return num_none

def assert_array_dimension(array, dimensions):
    '''Make sure arrays are the right dimensions
    '''
    if len(array.shape) != dimensions:
        raise ValueError('You must pass in a {}-D array!'.format(dimensions))

def embed_to_pca(EFILE, OUTFILE):
    """build pca model with embed file
    """
    Ef=pd.read_csv(EFILE,sep=' ',header=None).dropna(axis=1).transpose()
    Ef.columns=['x'+str(i) for i in Ef.columns]
    xf=Ef#.assign(IF=dx.ido)

    pca = PCA(n_components=2).fit(xf)
    ef=pca.fit_transform(xf)
    pd.DataFrame(ef).to_csv(OUTFILE,header=None,index=None)
    return ef

def replace_nan(df):
    """replace nans in a df and fill with empty string
    """
    df = df.replace('nan',np.nan).fillna('')
    return df


#!/usr/bin/env python3

import argparse

def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument('out')
    args = parser.parse_args()

    with open(args.out, 'w') as f:
        f.write('fn main() { println!("I prefer tarnish, actually.") }')

if __name__ == "__main__":
    main()

test cases/rust/11 generated main/gen.py


import cv2
import numpy as np

image = cv2.imread('images/indian_flag.png')

#reading pixel value as (0,0)
(b, g, r) = image[0, 0]
print("Pixel at (0, 0) - Red: {}, Green: {}, Blue: {}".format(r, g, b))

#replacing pixel value as (0,0) with 255
image[0, 0] = (0, 0, 255)
(b, g, r) = image[0, 0]
print("Pixel at (0, 0) - Red: {}, Green: {}, Blue: {}".format(r, g, b))

#cropping corner part from image
corner = image[0:100, 0:100]
cv2.imshow("Corner", corner)

#replacing corner pixles
image[0:100, 0:100] = (2255, 0, 255)

cv2.imshow('Indian Flag', image)
cv2.waitKey(0)


#!/usr/bin/env python3
import numpy as np
from stemming.porter2 import stem

vocab = []
vocab_index = {}

f = open('data/vocab_full.txt')
a = 0
for i, line in enumerate(f):
    print("line {}".format(i))
    strip = line.strip().split(" ")
    if len(strip) == 2:
        stemming = stem(strip[1])
        if vocab_index.get(stemming) is None:
            vocab.append(stemming)
            vocab_index[stemming] = a
            a += 1

np.save('vocab_full', vocab)
np.save('vocab_index_full', vocab_index)

neg = np.zeros((len(vocab),1))
pos = np.zeros((len(vocab),1))

f_pos = open('data/train_pos_full.txt')
for i, line in enumerate(f_pos):
    print("line {}", i)
    words = [t for t in line.strip().split()]
    for w in words:
        w = stem(w)
        index = vocab_index.get(w)
        if vocab_index.get(w) is not None:
            pos[index] += 1

f_neg = open('data/train_neg_full.txt')
for i, line in enumerate(f_neg):
    print("line {}", i)
    words = [t for t in line.strip().split()]
    for w in words:
        w = stem(w)
        index = vocab_index.get(w)
        if index is not None:
            neg[index] += 1

np.save('pos_full', pos)
np.save('neg_full', neg)


# -*- coding: utf-8 -*-

"""
***************************************************************************
    VariableDistanceBuffer.py
    ---------------------
    Date                 : August 2012
    Copyright            : (C) 2012 by Victor Olaya
    Email                : volayaf at gmail dot com
***************************************************************************
*                                                                         *
*   This program is free software; you can redistribute it and/or modify  *
*   it under the terms of the GNU General Public License as published by  *
*   the Free Software Foundation; either version 2 of the License, or     *
*   (at your option) any later version.                                   *
*                                                                         *
***************************************************************************
"""

__author__ = 'Victor Olaya'
__date__ = 'August 2012'
__copyright__ = '(C) 2012, Victor Olaya'

# This will get replaced with a git SHA1 when you do a git archive

__revision__ = '$Format:%H$'

from qgis.core import QGis

from processing.core.GeoAlgorithm import GeoAlgorithm
from processing.core.parameters import ParameterVector
from processing.core.parameters import ParameterBoolean
from processing.core.parameters import ParameterNumber
from processing.core.parameters import ParameterTableField
from processing.core.outputs import OutputVector
import Buffer as buff
from processing.tools import dataobjects

class VariableDistanceBuffer(GeoAlgorithm):

    INPUT = 'INPUT'
    OUTPUT = 'OUTPUT'
    FIELD = 'FIELD'
    SEGMENTS = 'SEGMENTS'
    DISSOLVE = 'DISSOLVE'

    def defineCharacteristics(self):
        self.name, self.i18n_name = self.trAlgorithm('Variable distance buffer')
        self.group, self.i18n_group = self.trAlgorithm('Vector geometry tools')

        self.addParameter(ParameterVector(self.INPUT,
                                          self.tr('Input layer'), [ParameterVector.VECTOR_TYPE_ANY]))
        self.addParameter(ParameterTableField(self.FIELD,
                                              self.tr('Distance field'), self.INPUT))
        self.addParameter(ParameterNumber(self.SEGMENTS,
                                          self.tr('Segments'), 1, default=5))
        self.addParameter(ParameterBoolean(self.DISSOLVE,
                                           self.tr('Dissolve result'), False))

        self.addOutput(OutputVector(self.OUTPUT, self.tr('Buffer')))

    def processAlgorithm(self, progress):
        layer = dataobjects.getObjectFromUri(self.getParameterValue(self.INPUT))
        dissolve = self.getParameterValue(self.DISSOLVE)
        field = self.getParameterValue(self.FIELD)
        segments = int(self.getParameterValue(self.SEGMENTS))

        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
            layer.pendingFields().toList(), QGis.WKBPolygon, layer.crs())

        buff.buffering(progress, writer, 0, field, True, layer, dissolve,
                       segments)


from keras.models import Sequential
from keras.layers.core import Dense, Activation
# from keras.optimizers import SGD, RMSprop
from keras.utils import np_utils
# Keras is a high-level neural networks API, written in Python and
#capable of running on top of TensorFlow, CNTK, or Theano.
# TensorFlow: a machine learning framework

# import matplotlib.pyplot as plt
import numpy as np
# from scipy import misc

import pandas # Python data analysis library

class neuralnet(object):
    '''defines a new object called "neuralnet", which is the output of the loadData() function.
    3 optional paramters: batch_size, nb_classes, nb_epoch
    '''

    def __init__(self,batch_size=100,nb_classes=3,nb_epoch=100):
       # Some generic parameters for the learning process
        self.batch_size = batch_size
        self.nb_classes = nb_classes
        self.nb_epoch   = nb_epoch
        self.loadData()

    def loadData(self,path):
        '''
        Takes path to directory containing spe files.
   	  Loads data from file into object.
   	  trainfile='train.txt',testfile='test.txt',labels_train='labels_train.txt',labels)test='labels_test.txt'
   	  '''
        #,trainfile='train.txt',testfile='test.txt',labels_train='labels_train.txt',labels_test='labels_test.txt'):

        import SPEtoCSVml
        import os
        spefilenames = []
        for root,dirs,files in os.walk(path):
            for file in files:
                if file.endswith('.spe'):
                    spefilenames.append(file)
        x = SPEtoCSVml.allSPEtoCSV(spefilenames)
        trainfile = x[0]
#        wrongsizefiles = x[1]

        self.train = pandas.read_csv(trainfile,header=None)
        self.test = pandas.read_csv(testfile,header=None)
        # creates attributes train and test (type DataFrame) from trainfile.csv and testfile.csv

        # pandas.read_csv returns DataFrame([data, index, columns, dtype, copy])
        #Two-dimensional size-mutable, potentially heterogeneous tabular data structure
        #with labeled axes (rows and columns).

        # header : int or list of ints. If file contains no header row,
        #then you should explicitly pass header=None.

        self.train = self.train.as_matrix()
        self.test = self.test.as_matrix()
        # .as_matrix method converts the dataframe to its Numpy-array representation.
        # i.e. >>> type(self.train) --> numpy.ndarray
        print(self.train.shape)

        self.train = self.train.transpose()
        self.test = self.test.transpose()
        # why transpose?

        self.labels_train = pandas.read_csv(labels_train,header=None)
        self.labels_test = pandas.read_csv(labels_test,header=None)
        # what're labels for?

        self.labels_train = self.labels_train.as_matrix()
        self.labels_test = self.labels_test.as_matrix()

        self.labels_train = np.resize(self.labels_train, self.labels_train.shape[1])
        self.labels_test = np.resize(self.labels_test, self.labels_test.shape[1])
        #

        self.labels_train = np_utils.to_categorical(self.labels_train,self.nb_classes)
        self.labels_test  = np_utils.to_categorical(self.labels_test,self.nb_classes)
        # np_utils ?

    def buildModel(self):
        self.model = Sequential()
        # The Sequential model is a linear stack of layers.
        # You can create a Sequential model by passing a list of layer instances to the constructor
        self.model.add(Dense(80,input_shape=(80,)))
        # the first layer in a Sequential model (and only the first, because following layers can do automatic shape inference)
        #needs to receive information what input shape to expect.
        self.model.add(Activation('relu'))
        self.model.add(Dense(40))
        self.model.add(Activation('relu'))
        self.model.add(Dense(3))
        self.model.add(Activation('softmax'))
        sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
        self.model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=["accuracy"])

    def fit(self):
        self.model.fit(self.train,self.labels_train,
        batch_size=self.batch_size,
        nb_epoch=self.nb_epoch,
        verbose=0,
        validation_data = (self.test,self.labels_test))

    def showResults(self):
        score = self.model.evaluate(self.test,self.labels_test)
        print('Test score:', score[0])
        print('Test accuracy:', score[1])
        return score[1]


def sum_of_proper_divisor(n):
    sum=0
    x=[]
    lim= (n//2)+1       #optimizing the loop by checking divisiors till half + 1
    for i in range(1,lim):
        if n % i == 0:
            sum+=i
            x.append(i)
    if n != sum:
        return sum
    else:
        return -1

if __name__ == "__main__":

    sum=0
    memo=[]
    for n in range(10000):  #put limit value here.
        if n not in memo:           # using memoziation concept to skip calculations on a amicable pair.
            #print(n)
            x = sum_of_proper_divisor(n)
            if x != -1:
                y = sum_of_proper_divisor(x)
            else:
                continue
            if y == n:
                sum += n
                sum += x
                memo.append(x)
    print("sum of amicable numbers from 1 to 10000", str(sum))


from astropy.io import fits
from astropy.wcs import WCS
import numpy as np
import matplotlib.pyplot as plt

# Finish the new Observer class!
class Observer():
    '''
    This class creates an artificial night sky observer.
    '''

    # This function will get called automatically
    # when a new "observer" is created
    def __init__(self,im1_filename,im2_filename):
        '''
        When initializing the observer, the "red" image should be given
        as the first input argument and the "ir" image should be the second input
        '''
        self.im1_filename = im1_filename
        self.im2_filename = im2_filename
        self.load_images(im1_filename,im2_filename)

    def load_images(self,im1_filename,im2_filename):
        image_data_red = fits.getdata(self.im1_filename)
        image_data_ir = fits.getdata(self.im2_filename)
        image = {}
        image[self.im1_filename] = image_data_red
        image[self.im2_filename] = image_data_ir
        return image

    def make_composite(self,im_dict):
        '''
        This function is incomplete! Make sure to finish it and
        then update this docstring to explain what the function does!
        '''
        ir_hud_list = fits.open(self.im2_filename)[0]
        wcs = WCS(ir_hud_list.header)
        # Define the array for storing RGB values
        rgb = np.zeros((im_dict[self.im1_filename].shape[0],im_dict[self.im1_filename].shape[1],3))

        # Define a normalization factor for our denominator using the R filter image
        norm_factor = im_dict[self.im1_filename].astype("float").max()

        # Compute the red channel values and then clip them to ensure nothing is > 1.0
        rgb[:,:,0] = 1.5 * (im_dict[self.im2_filename].astype("float")/norm_factor)
        rgb[:,:,0][rgb[:,:,0] > 1.0] = 1.0
            # Define the array for storing RGB values
        rgb = np.zeros((im_dict[self.im1_filename].shape[0],im_dict[self.im1_filename].shape[1],3))

    # Define a normalization factor for our denominator using the R filter image
        norm_factor = im_dict[self.im1_filename].astype("float").max()

    # Compute the red channel values and then clip them to ensure nothing is > 1.0
        rgb[:,:,0] = (im_dict[self.im2_filename].astype("float")/norm_factor) * 1.5
        rgb[:,:,0][rgb[:,:,0] > 1.0] = 1.0

        rgb[:,:,1] = ((im_dict[self.im2_filename].astype("float")+ im_dict[self.im1_filename].astype("float"))/2)/norm_factor

        rgb[:,:,1][rgb[:,:,1] > 1.0] = 1.0

        rgb[:,:,2] = (im_dict[self.im2_filename].astype("float"))/norm_factor

        rgb[:,:,2][rgb[:,:,2] > 1.0] = 1.0

        plt.subplot(projection = wcs)
        plt.xlabel('Right Acension')
        plt.ylabel('Declination')
        plt.grid(color='white')
        plt.imshow(rgb)

    def calc_stats(self):
        image_dict = np.array(list(self.load_images(self.im1_filename,self.im2_filename)[self.im1_filename]))
        image_dict = np.append(image_dict,np.array(list(self.load_images(self.im1_filename,self.im2_filename)[self.im2_filename])))
        mean = image_dict.mean()
        std = image_dict.std()
        return mean, std


import json
import os
import hashlib
import random
import string
from time import time

# path to chain dir
chain_dir = os.curdir + '/chain/'
files = os.listdir(chain_dir)
files = sorted(int(i) for i in files)

def get_hash(file_name):          # hash def \__0__/
	file = open(chain_dir + file_name, 'rb').read()# указываем имя файла, который мы открываем для чтения с указанием папки
	return hashlib.sha256(file).hexdigest()

def check_integrity():
    for file in files[1:]:
        h = json.load(open(chain_dir + str(file)))['hash']
        actual_hash = get_hash(str(file - 1))

        prev_block = str(file -1)

        if h == actual_hash:
            res = 'kk'
        else:
            res = 'corrupted'
        print('block {} is {}'.format(prev_block, res))

def create_account(wallet):
	userid = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(16))
	user = {'index': int(str(files[-1] + 1)),
			'timestamp': time(),
			'userid': userid,
			'wallet': wallet
	}
	with open(chain_dir + str(files[-1] + 1), 'w') as file:
		json.dump(user, file, indent=4, ensure_ascii=False)

def write_block(sender, amount, message, recipient, prev_hash=''):
	prev_file = files[-1]  # последний блок, который находится в папке chain
	file_name = str(prev_file + 1)  # называем след.файл символьным значением
	prev_hash = get_hash(str(prev_file))  # олучаем хэш предыдущего файла

	for file in files[1:]:
		u = json.load(open(chain_dir + str(file)))['userid']
		if u == sender:
			sw = json.load(open(chain_dir + str(file)))['wallet']
			sender_wallet = sw - amount

	for file in files[1:]:
		u = json.load(open(chain_dir + str(file)))['userid']
		if u == recipient:
			rw = json.load(open(chain_dir + str(file)))['wallet']
			recipient_wallet = rw + amount

	block = {'index': int(str(files[-1] + 1)),
			'timestamp': time() ,
			 'sender' : sender,
			 'amount' : amount,
			 'message' : message,
			 'recipient' : recipient,
			 'sender_wallet': sender_wallet,
			 'recipient_wallet': recipient_wallet,
			 'hash': prev_hash}

	with open(chain_dir + file_name, 'w') as file:
		json.dump(block, file, indent=4, ensure_ascii=False)

def main():

	#write_block(input(), int(input()), input(), input())
	# check_integrity()
	create_account(int(input()))

if __name__ == '__main__':
	main()


from django.core.management.base import BaseCommand, CommandError
from django.conf import settings

from subprocess import check_output, call
from optparse import make_option
import re,os

from dom4gameserver.models import Game
from dom4gameserver.helpers import findgameprocess

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('-g', '--game',
            dest='gamename',
            action='store',
            default='all',
            metavar='GAME',
            help='Game name to check'),
        )

    def handle(self, *args, **options):
        if options['gamename'] != 'all':
            self.stdout.write('Unimplemented Feature.')
        else:
            for gg in Game.objects.filter(status__in = ['PRETENDER','RUNNING']):
                if not findgameprocess(gg):
                    self.stdout.write("I didn't find game: %s.   Attempting to Restart..." % gg.name)
                    call(["sudo", '-u',settings.DOM4_USER, os.path.join(settings.DOM4GAME_DIR,gg.servername)],cwd=settings.DOM4GAME_DIR)


import torch
import numpy as np
import sys
import pandas as pd

# OPTIONAL: if you want to have more information on what's happening, activate the logger as follows
import logging
logging.basicConfig(level=logging.INFO)

use_cuda = torch.cuda.is_available()
device = torch.device('cuda:0' if use_cuda else 'cpu')

path_to_wsc = '../../data/wsc_data/enhanced.tense.random.role.syn.voice.scramble.freqnoun.tsv'
wsc_datapoints = pd.read_csv(path_to_wsc, sep='\t')

#wsc_file = open(path_to_wsc, 'r')
#wsc_datapoints = wsc_file.readlines()

def insert_highlight(text_tokenized, pronoun_index):
    text_tokenized.insert(pronoun_index, " ")
    text_tokenized.insert(pronoun_index + 2, " ")

    return text_tokenized

def find_sub_list(sl,l):
    results=[]
    sll=len(sl)
    for ind in (i for i,e in enumerate(l) if e.lower() == sl[0].lower()):
        item = str(l[ind:ind+sll]).lower()
        sub_item = str(sl).lower()
        if  item == sub_item:
            results.append((ind,ind+sll))
    return results

def replace_pronoun(tokenized_text, pronoun_index, tokenized_option):
    tokenized_text = tokenized_text[:pronoun_index] + tokenized_option + tokenized_text[pronoun_index:]
    new_pronoun_index = pronoun_index + len(tokenized_option)
    tokenized_text.pop(new_pronoun_index)
    return tokenized_text

counter = 0
limit_of_dps = 60
split_counter = 0
filename = sys.argv[1]

for q_index, dp_split in wsc_datapoints.iterrows():
        if counter % limit_of_dps == 0 or counter == 0:

            split_counter = split_counter + 1
            fsplitname = str(filename) + "." + str(split_counter)
            if counter != 0:
                f.close()
            f = open(fsplitname, 'w')

        correct_answer = dp_split['correct_answer'].strip()
        pronoun = dp_split['pron'].strip()
        pronoun_index_orig =  dp_split['pron_index_scrambled']

        answer_A = dp_split['answer_a'].strip()
        answer_B = dp_split['answer_b'].strip()
        text = dp_split['text_scrambled'].strip()

        if  pronoun_index_orig != '-' and  pronoun_index_orig:
            counter = counter + 1
            text_tokenized = text.split()
            pronoun_index_orig = int(pronoun_index_orig)

            #match pron
            matched_pronouns = find_sub_list([pronoun], text_tokenized)
            first_indices = np.array([mp[0] for mp in  matched_pronouns])
            correct_idx = (np.abs(first_indices - pronoun_index_orig)).argmin()
            pronoun_index_text = matched_pronouns[correct_idx][0]

            text_tokenized_highlighted = insert_highlight(text_tokenized, pronoun_index_text)

            label = "l: " + str(counter)
            type = "t: radio"
            question = "q: " + ' '.join(text_tokenized_highlighted)
            answera = "- " + answer_A
            answerb = "- " + answer_B

            f.write(label)
            f.write('\n')
            f.write(type)
            f.write('\n')
            f.write(question)
            f.write('\n')
            f.write(answera)
            f.write('\n')
            f.write(answerb)
            f.write('\n')
            f.write('\n')

            #print(label)
            #print(type)
            #print(question)
            #print(answera)
            #print(answerb)
            #print('\n')


from sys import stdin

def ferry(carros,len_ferry):
    car_left=[int(carros[i][0]) for i in range(len(carros)) if carros[i][1]=="left"]
    car_right=[int(carros[i][0]) for i in range(len(carros)) if carros[i][1]=="right"]
    cl=len(car_left)
    cr=len(car_right)
    cruce=0
    while cl or cr:
        temp=len_ferry
        if cl==0 and cr>0:
            cruce+=1
        while cl:
            if temp-car_left[0]<0:
                cruce+=1
                break
            else:
                temp-=car_left[0]
                del car_left[0]
                cl-=1
                if cl==0:
                    cruce+=1
        temp=len_ferry
        if cr==0 and cl>0:
            cruce+=1
        while cr:
            if temp-car_right[0]<0:
                cruce+=1
                break
            else:
                temp-=car_right[0]
                del car_right[0]
                cr-=1
                if cr==0:
                    cruce+=1
    return cruce

def main():
    casos=int(stdin.readline().strip())
    for x in range(casos):
        len_ferry,m=map(int,stdin.readline().strip().split())
        len_ferry=len_ferry*100
        carros=[]
        for y in range(m):
            inst=tuple(stdin.readline().strip().split())
            carros.append(inst)
        print(ferry(carros,len_ferry))
main()


"""
age = "I am 26"
print(f"I am {26} years old.") # 'f' stands for format.
print(5/2)
 """
#Global Variables
calculate_to_unit = 24 # This is python naming convention
name_of_unit = "hours"
#Global Variables

"""
print(f"24 days is = {calculate_to_unit} {name_of_unit}")
print(f"30 days is = {calculate_to_unit} {name_of_unit}")
print(f"365 days is = {calculate_to_unit} {name_of_unit}")
"""

"""
def days_to_units(name_of_days): #def is used to define a function(method).
    print(f"{name_of_days} days is = {name_of_days*calculate_to_unit} {name_of_unit}")
"""
# Sting er modhe variable use korte hole () use korte hbe

def days_to_units(name_of_days): #def is used to define a function(method).
    return (f"{name_of_days} days is = {name_of_days * calculate_to_unit} {name_of_unit}")

"""
days_to_units(20) # To run a function you have to call by giving () after the variable
days_to_units(30)
days_to_units(50)
"""
"""
def input_validation():
    if user_input.isdigit():#to check if the input value is int or text or something else
        user_input_number = int(user_input)
        if user_input_number > 0:
            calculated_hours = days_to_units(user_input_number)
            print(calculated_hours)
        elif user_input_number == 0:
            print("Given value greater than 0 days")
        #python can work without else statement
    else:
        print("value is not an integer")
"""

#try/except = try/exception = try/catch
def input_validation():
    try:
        user_input_number = int(num_of_days)
        if user_input_number > 0:
            calculated_hours = days_to_units(user_input_number)
            print(calculated_hours)
        elif user_input_number == 0:
            print("Given value greater than 0 days")
        #python can work without else statement
    except:
        print("value is not an integer")

user_input = ""
#----------------list-------------------------------

"""
while user_input != "exit":
    user_input = input("Please enter the number of days.\n") #Always take input as String
    # For clean code every functio should be modularized.
    #for num_of_days in user_input.split():# by default space is used to separate integers
    for num_of_days in user_input.split(','):# by comma(,) is used to separate integers
        input_validation() """

#----------------------Set---------------------------
while user_input != "exit":
    user_input = input("Please enter the number of days.\n") #Always take input as String exit
    for num_of_days in set(user_input.split(',')):# by comma(,) is used to separate integers
        input_validation()