from django.contrib.auth.models import Permission
from django import forms
from .models import Proyecto, ProyectUser, RolProyecto, Sprint, UserStory, Daily
from sso.models import User
from django import forms
from django.forms.models import inlineformset_factory

class AgregarRolProyectoForm(forms.ModelForm):
    """
    Form para agregar un rol al proyecto.
    Se despliega la lista de permisos para que puedan seleccionarse los roles.
    """
    def __init__(self, *args, **kwargs):
        super(AgregarRolProyectoForm, self).__init__(*args, **kwargs)
        self.fields['permisos'] = forms.MultipleChoiceField(widget=forms.CheckboxSelectMultiple,
                                                            choices=[(p.id, p.name) for p in Permission.objects.all() if
                                                                     p.codename.startswith('p_') and not p.codename.startswith('pg_') and not p.codename.startswith('p_acceder_proyecto')])
    class Meta:
        model = RolProyecto
        fields = ['nombre','permisos']

class UserAssignRolForm(forms.ModelForm):
    """
    Form para asignar un rol a los participantes del proyecto.
    """
    class Meta:
        model = RolProyecto
        fields = ['participantes']

    def __init__(self, *args, **kwargs):
        proyecto_id = kwargs.pop('pk_proy',None)
        proyecto = Proyecto.objects.get(id=proyecto_id)
        super(UserAssignRolForm, self).__init__(*args, **kwargs)
        self.fields['participantes'] = CustomUserMCF(queryset= proyecto.equipo.all(),
            widget=forms.CheckboxSelectMultiple
        )
        self.fields['participantes'].required = False

class CustomUserMCF(forms.ModelMultipleChoiceField):
    """ Un ModelMultipleChoiceField custom """
    def label_from_instance(self, member):
        return "%s" % member.first_name + " " + member.last_name

class ProyectoEditForm(forms.ModelForm):
    """
    Form de proyecto que recibe los parámetros asociados al nombre, duración del sprint, fecha de incio y fin
    y el equipo encargado del proyecto.
    """
    class Meta:
        model = Proyecto
        fields = ["nombreProyecto", "fechaInicio", "fechaFin"]

class ProyectoCreateForm(forms.ModelForm):
    """
    Form de proyecto que recibe los parámetros asociados al nombre, duración del sprint, fecha de incio y fin
    y el equipo encargado del proyecto.
    """
    class Meta:
        model = Proyecto
        fields = ["nombreProyecto", "fechaInicio", "fechaFin", "equipo"]

    def __init__(self, *args, **kwargs):
        user = kwargs.pop('user_id',None)
        user = User.objects.get(pk = user)
        super(ProyectoCreateForm,self).__init__(*args, **kwargs)
        self.fields['equipo'] = forms.MultipleChoiceField(
        widget=forms.CheckboxSelectMultiple,
        choices=[(p.id, "%s" % p.first_name + " " + p.last_name) for p in User.objects.exclude(first_name__isnull=True).exclude(first_name__exact='').exclude(pk=user.pk) if p.has_perm('sso.pg_is_user')],
        required=False
    )

class ProyectoFinalizarForm(forms.ModelForm):
    """ Form para finalizar un Proyecto, no se muestran realmente campos pero el form es nescesario igual. """
    def __init__(self, *args, **kwargs):
        super(ProyectoFinalizarForm, self).__init__(*args, **kwargs)
        self.fields['estado_de_proyecto'].required = False
    class Meta:
        model = Proyecto
        fields = ["estado_de_proyecto",]

class ImportarRolProyectoForm(forms.Form):
    """
    Form para importar roles de proyecto. Se seleccionan de entre todos los proyectos existentes
    en el sistema y se extraen sus roles para desplegarse en pantalla.
    """
    class Meta:
        fields = ["roles"]

    def __init__(self, *args, **kwargs):
        proyecto_id = kwargs.pop('pk_proy',None)
        super(ImportarRolProyectoForm, self).__init__(*args, **kwargs)
        self.fields['roles'] = forms.MultipleChoiceField(
        widget=forms.CheckboxSelectMultiple,
        choices=[(p.id, p.nombre) for p in RolProyecto.objects.exclude(proyecto__id=proyecto_id)]
    )

class AgregarParticipanteForm(forms.Form):
    """
    Form para agregar Participantes al Proyecto
    """
    class Meta:
        model = Proyecto
        fields = ['equipo']

    def __init__(self, *args, **kwargs):
        proyecto = kwargs.pop('instance',None)
        super().__init__(*args, **kwargs)
        self.fields['equipo'] = forms.MultipleChoiceField(
        widget=forms.CheckboxSelectMultiple,
        choices=[(p.id, "%s" % p.first_name + " " + p.last_name) for p in User.objects.exclude(first_name__isnull=True).exclude(first_name__exact='').exclude(pk=proyecto.owner.pk).exclude(proyecto__id=proyecto.pk) if p.has_perm('sso.pg_is_user')]
    )

class DesarrolladorCreateForm(forms.ModelForm):
    """ Form para administrar el equipo de desarrolladores de un sprint. """

    def __init__(self, *args, **kwargs):
        proyect_id = kwargs.pop('proyecto',None)
        proyecto = Proyecto.objects.get(pk=proyect_id)
        super().__init__(*args, **kwargs)
        self.fields['usuario'] = forms.ModelChoiceField(
            empty_label="Opciones",
            queryset=proyecto.equipo.all() | User.objects.filter(pk=proyecto.owner.pk),
        )
        self.fields['horas_diarias'].required = True
        self.fields['usuario'].required = True
    class Meta:
        model = ProyectUser
        fields = ['usuario','horas_diarias']

class PermisoSolicitudForm(forms.Form):
    """ Form para solicitar más permisos al owner del proyecto. """
    asunto = forms.CharField(label='Asunto de la solitud', max_length=100, required=True)
    body = forms.CharField(widget=forms.Textarea,label='Solicitud',help_text='Especifique, que tipo de acceso o permisos necesita. Explique que acciones quiere hacer.',required=True)

class SprintCrearForm(forms.ModelForm):
    """ Form utilizado para la creación de un sprint. Se llenan los campos duración del Sprint y fechafin """
    class Meta:
        model = Sprint
        fields = ['duracionSprint']

class SprintModificarForm(forms.ModelForm):
    """ Form para modificar la duración del sprint """
    class Meta:
        model = Sprint
        fields = ['duracionSprint']

class SprintFinalizarForm(forms.ModelForm):
    """ Form para la finalización de los sprints """

    def __init__(self, *args, **kwargs):
        super(SprintFinalizarForm, self).__init__(*args, **kwargs)
        self.fields['estado_de_sprint'].required = False
    class Meta:
        model = Sprint
        fields = ['estado_de_sprint']

class UserstoryAprobarForm(forms.ModelForm):
    """ From para aprobar un user story temporal. No muestra campos pero es nescesario. """
    def __init__(self, *args, **kwargs):
        super(UserstoryAprobarForm, self).__init__(*args, **kwargs)
        self.fields['estado_aprobacion'].required = False
    class Meta:
        model = UserStory
        fields = ['estado_aprobacion']

EquipoFormset = inlineformset_factory(Sprint, ProyectUser,fields=('usuario','horas_diarias',),form=DesarrolladorCreateForm,can_delete=True)
class AgregarUserStoryForm(forms.ModelForm):
    """
    Form para crear un user story nuevo, renderiza los campos de prioridad, nombre y descripción
    """
    def __init__(self, *args, **kwargs):
        super(AgregarUserStoryForm, self).__init__(*args, **kwargs)
        self.fields['prioridad_user_story'] = forms.ChoiceField(
            widget= forms.RadioSelect,
            choices=[('B','Baja'),('A','Alta'),('M','Media'),('E','Emergencia')]
        )

    class Meta:
        model = UserStory
        fields = ['nombre','descripcion','prioridad_user_story']

class UserStoryAssingForm(forms.ModelForm):
    """ Form utilizado por el scrum master para asignar un user story a un desarrollador y estimar el tiempo de completar el user story """
    def __init__(self, *args, **kwargs):
        sprint_id = kwargs.pop('sprint_id',None)
        sprint = Sprint.objects.get(pk=sprint_id)
        super().__init__(*args, **kwargs)
        self.fields['encargado'] = forms.ModelChoiceField(
            empty_label="Desarrollador",
            queryset=sprint.sprint_team.all(),
        )
        self.fields['tiempo_estimado_scrum_master'].required = True
    class Meta:
        model = UserStory
        fields = ['tiempo_estimado_scrum_master','encargado','tiempo_estimado_dev']

class UserStoryDevForm(forms.ModelForm):
    """ Form utilizado por el encargado asignado al user story para estimar el tiempo de duración de completar el user story """
    def __init__(self, *args, **kwargs):
        sprint_id = kwargs.pop('sprint_id',None)
        super().__init__(*args, **kwargs)
    class Meta:
        model = UserStory
        fields = ['tiempo_estimado_dev']

class QaForm(forms.Form):
    """ Form de qa. Se llena con un comentario de evaluación que es requerido y que se envia posteriormente al encargado del user story """
    comentario = forms.CharField(widget=forms.Textarea,label='Comentario',help_text='De un comentario o recomendación.',required=True)

class DailyForm(forms.ModelForm):
    """
    Form para crear y modificar un Daily. Se muestran los campos de duración, del comentario de impedimientos y del comentario de progresos.
    """
    def __init__(self, *args, **kwargs):
        super(DailyForm, self).__init__(*args, **kwargs)

    class Meta:
        model = Daily
        fields = ['fecha','duracion','impedimiento_comentario','progreso_comentario']

class ReasignarForm(forms.ModelForm):
    """ Form para reasignar un user story a otro dev del sprint. Solo se cambiará el campo encargado del user story. """
    def __init__(self, *args, **kwargs):
        sprint_id = kwargs.pop('sprint_id',None)
        sprint = Sprint.objects.get(pk=sprint_id)
        pk_proy = kwargs.pop('pk_proy',None)
        proyecto = Proyecto.objects.get(pk=pk_proy)
        user = User.objects.filter(pk=proyecto.owner.pk)
        super().__init__(*args, **kwargs)
        self.fields['encargado'] = forms.ModelChoiceField(
            empty_label="Desarrollador",
            queryset=sprint.sprint_team.all(),
        )
    class Meta:
        model = UserStory
        fields = ['encargado']

class IntercambiarDevForm(forms.ModelForm):
    """ Form para intercambiar un desarrollador de un sprint con otro asociado al proyecto. Solo se cambai el usuario para cambiar el ProyectUser. """
    def __init__(self, *args, **kwargs):
        pk_proy = kwargs.pop('pk_proy',None)
        proyecto = Proyecto.objects.get(pk=pk_proy)
        user = User.objects.filter(pk=proyecto.owner.pk)
        super().__init__(*args, **kwargs)
        self.fields['usuario'] = forms.ModelChoiceField(
            empty_label="Desarrollador",
            queryset=proyecto.equipo.all() | user,
        )
    class Meta:
        model = ProyectUser
        fields = ['usuario']


from flask import Flask,render_template,redirect,request
from mysqlconnection import connectToMySQL

# from mysqlconnection import connectToMySQL

app = Flask(__name__)

@app.route('/')
def index():
    mysql= connectToMySQL('users_db')
    users = mysql.query_db ('SELECT * FROM users;')
    print(users)
    return render_template("index.html", all_users=users)

@app.route('/create', methods=['POST'])
def add_users_to_db():
    query = "INSERT INTO users ( first_name , last_name , email, created_at, updated_at) VALUES (%(first_name)s,%(last_name)s, %(email)s,NOW(),NOW());"
    data = {
    'first_name': request.form['first_name'],
    'last_name': request.form['last_name'],
    'email': request.form['email']
    }
    bd = connectToMySQL('users_db')
    bd.query_db(query,data)
    print(bd)
    return redirect('/')

@app.route('/create_page')
def creat_page():
    return render_template('create.html')

if __name__ == "__main__":
    app.run(debug=True)


# *****************************************************************************
# ПРОЕКТ: ISS Docking Simulator
# ФАЙЛ: game.py
# ОПИСАНИЕ: основной модуль проекта
# *****************************************************************************

# -- ИМПОРТ БИБЛИОТЕК
import pygame, os, datetime, math as Math
from pygame.locals import *
from padlib import *

os.environ['SDL_VIDEO_WINDOW_POS'] = "%d,%d" % (50,50) # зададим координаты вывода главного окна

# ---- ФУНКЦИИ
def refineDest(sh, sv):
	a = set()
	if round(sh, 1) > 0.0:
		a.add("ВОСТОК")
	if round(sh, 1) < 0.0:
		a.add("ЗАПАД")
	if round(sv, 1) > 0.0:
		a.add("ЮГ")
	if round(sv, 1) < 0.0:
		a.add("СЕВЕР")
	return a

# -- ИНИЦИАЛИЗАЦИЯ
random.seed()
pygame.init()
screen = pygame.display.set_mode((800, 600)) # главная поверхность
pygame.display.set_caption('IIS Docking simulator')

pygame.font.init()
font_dot = pygame.font.Font('square_dot_digital-7.ttf', 30)

# -- ЗАДАЕМ НАЧАЛЬНЫЕ НАСТРОЙКИ
# загрузка картинок
img_scr_intro = pygame.image.load("Scr_Intro.jpg")
img_scr_task = pygame.image.load("Scr_Task.jpg")
img_progressm = pygame.image.load("img_progressm.png").convert_alpha()
img_background = pygame.image.load("background.jpg").convert()
img_brokenglass = pygame.image.load("img_brokenglass.png").convert_alpha()

# получить системный таймер для того, что бы задать FPS=25 кадров в секунду
clock = pygame.time.Clock()
fps = 25

# инициализация переменных
mainExit = False # флаг выхода из главного цикла
game_state = 0 # текущее состояние игры. То что будет обрабатывать главный цикл зависит от этой переменной
"""
0 - начальный экран №1 (введение)
1 - начальный экран №2 (задача)
2 - игровой экран (сближение со станцией)
3 - финальный экран (сигнал потерян / стыковка)

"""

dp_x, dp_y = random.randrange(100, 700), random.randrange(100, 500) # координаты Прогресса
dp_w, dp_h = 3119, 1276 # ширина и высота картинки Прогресса
dp_scale, dp_scale_step, dp_scale_stop = 10.0, 0.008, 0.5 # коэффициент уменьшения Прогресса. Используется для маштабирования картинки с целью симуляции приближения
dp_rotation, dp_rotation_step = random.randrange(0, 359), 0.2 # вращение Прогресса (0.2)
dp_scale_suspended = 0

# скорость корабля является линейной. По хорошему должна быть логарифмическая, но она для земного сопротивления по воздуху
speed_h, speed_v = 0.0, 0.0 # горизонтальная и вертикальная скорости
speed_max, speed_step = 9.0, 0.1 # максимальная скорость сближения, шаг увеличения скорости

# вычисляем движение в зависимости от появления
if dp_x < 400 and dp_y < 300: speed_h, speed_v = 1.0, 1.0
if dp_x < 400 and dp_y >= 300: speed_h, speed_v = 1.0, -1.0
if dp_x >= 400 and dp_y < 300: speed_h, speed_v = -1.0, 1.0
if dp_x >= 400 and dp_y >= 300: speed_h, speed_v = -1.0, -1.0

speed_direction = refineDest(speed_h, speed_v) # направление движения

iis_hit = False # True если Прогресс врезался в МКС
disp_interlaced = False

# константы
white = (255,255,255)
black = (0,0,0)
red = (255,0,0)
green = (0,255,0)
yellow = (255,255,0)

# расстояние между двумя точками по их координатам: от центра картинки до изображенного центра шлюза
docking_center = Math.sqrt(Math.pow((dp_w/2)-1604, 2) + Math.pow((dp_h/2)-705, 2))

# -- ОСНОВНОЙ ЦИКЛ ВЫПОЛНЕНИЯ ПРОГРАММЫ
while not mainExit:

	# обработка событий
	for event in pygame.event.get():
		# нажат крестик окна - выход из программы
		if event.type == QUIT:
			mainExit = True
		# если щелчок мыши или любой клавишей
		elif event.type == pygame.MOUSEBUTTONUP or event.type == pygame.KEYUP:
			if game_state < 2:
				game_state += 1
			elif game_state == 2:
				if event.key and event.key == K_SPACE:
					disp_interlaced = not disp_interlaced
				elif event.key and event.key == K_RETURN:
					if dp_scale_step != 0:
						dp_scale_suspended = dp_scale_step
						dp_scale_step = 0
					else:
						dp_scale_step = dp_scale_suspended

	# ОБРАБОТКА ВЫВОДА ВСЕХ ИЗОБРАЖЕНИЙ В ЭКРАННЫЙ БУФЕР
	if game_state == 0:
		""" основной экран """
		screen.blit(img_scr_intro, (0,0))

	elif game_state == 1:
		""" задача симуляции полета """
		screen.blit(img_scr_task, (0,0))

	elif game_state >= 2:
		""" симулятор """

	# если это игровой экран, задаем направление движения
		keys = pygame.key.get_pressed()
		if len(keys) != 0:
			if keys[K_LEFT]:
				if abs(speed_h) <= speed_max and dp_x < 800:
					speed_h += speed_step

			if keys[K_RIGHT]:
				if abs(speed_h) <= speed_max and dp_x > 0:
					speed_h -= speed_step

			if keys[K_UP]:
				if abs(speed_v) <= speed_max and dp_y < 600:
					speed_v += speed_step

			if keys[K_DOWN]:
				if abs(speed_v) <= speed_max and dp_y > 0:
					speed_v -= speed_step

			# ограничение на скорость
			if abs(speed_h) > speed_max:
				speed_h = speed_max * (-1 if speed_h < 0 else 1)
			if abs(speed_v) > speed_max:
				speed_v = speed_max * (-1 if speed_v < 0 else 1)

		# перемещение и вращение
		if dp_scale != dp_scale_stop:
			dp_x += speed_h
			dp_y += speed_v
			dp_rotation -= dp_rotation_step # вращение
			dp_rotation %= 360

		# ограничение на выход за пределы экрана
		if dp_x <= 0:
			dp_x = 0.0
			speed_h = 0
		if dp_y <= 0:
			dp_y = 0.0
			speed_v = 0
		if dp_x >= 800:
			dp_x = 800
			speed_h = 0.0
		if dp_y >= 600:
			dp_y = 600
			speed_v = 0.0

		speed_direction = refineDest(speed_h, speed_v)

		# вычисляем сближение
		dp_scale -= dp_scale_step
		if dp_scale < 1:
			dp_scale -= (dp_scale_step * 1.3)
		elif dp_scale < 2:
			dp_scale -= (dp_scale_step * 1.1)
		if dp_scale <= dp_scale_stop:
			# сближение закончено
			dp_scale = dp_scale_stop

		# очистили экран
		screen.fill(black)

		# поворачиваем Прогресс
		dp_rotated = pygame.transform.rotate(
				pygame.transform.scale(img_progressm, (int(dp_w/dp_scale), int(dp_h/dp_scale))),
				dp_rotation)
		#dp_rotated = pygame.transform.rotozoom(img_progressm, dp_rotation, 1/dp_scale)

		# высчитываем координаты
		r = docking_center / dp_scale

		# -- координаты вывода картинки
		xImg = dp_x - dp_rotated.get_rect().center[0]
		yImg = dp_y - dp_rotated.get_rect().center[1]

		# -- центр стыковки
		x0 = dp_x + r * Math.cos(Math.radians(57-dp_rotation))
		y0 = dp_y + r * Math.sin(Math.radians(57-dp_rotation))
		rc = pygame.Rect( int(x0-50/dp_scale), int(y0-50/dp_scale), int(100/dp_scale), int(100/dp_scale) )
		rc1 = pygame.Rect( int(x0-25/dp_scale), int(y0-25/dp_scale), int(50/dp_scale), int(50/dp_scale) )

		# отрисовываем Прогресс
		screen.blit(img_background, (0,0), (0,0,800,600))
		screen.blit(dp_rotated, (int(xImg), int(yImg)))

		# имитация чресстрочной развертки
		if disp_interlaced:
			for iy in range(0, 600, 2):
				pygame.draw.line(screen, (10,10,10), (0, iy), (800, iy), 1)

		# вычисляем находится ли перекрестие в допустимой зоне стыковки
		# а заодно определяем - успешно ли мы пристыковались или врезались
		# и выводим прямоугольник зоны стыковки
		if dp_scale < 5:
			if rc.collidepoint(400, 300):
				pygame.draw.rect(screen, green, rc, 2)
			else:
				pygame.draw.rect(screen, red, rc, 2)
		# вторая мишень
		if dp_scale < 2:
			if rc1.collidepoint(400, 300):
				pygame.draw.rect(screen, yellow, rc1, 1)
				if dp_scale == dp_scale_stop:
					iis_hit = False # успешно
					game_state = 3 # следующий игровой экран
			else:
				pygame.draw.rect(screen, red, rc1, 1)
				if dp_scale == dp_scale_stop:
					iis_hit = True # врезались
					game_state = 3 # следующий игровой экран

		# рисуем вид из камеры грузового корабля
		DashedLine(screen, white, black, (400, -13), (400, 599), 11) #верт линия
		DashedLine(screen, white, black, (-1, 300), (799, 300), 11) #гориз линия

		# выводим всякие типа служебные надписи для красоты
		fnt = font_dot.render("Ф44   СБЛИЖ         ПРИЧАЛ Т={}".format(
			       datetime.datetime.now().strftime("%H:%M:%S")), True, white)
		screen.blit(fnt, (95, 30))
		fnt = font_dot.render("ЛСК               ГСО 1234", True, white)
		screen.blit(fnt, (95, 60))
		fnt = font_dot.render("СКОРОСТЬ: {}H {}V".format(round(speed_h, 2), round(speed_v, 2)), True, white)
		screen.blit(fnt, (30, 110))
		fnt = font_dot.render("КУРС: {}".format( ("-".join(d for d in speed_direction)) if len(speed_direction) != 0 else "НЕТ"), True, white)
		screen.blit(fnt, (30, 140))
		fnt = font_dot.render("СБЛИЖЕНИЕ: {}".format(-1 * round(dp_scale - 1, 3)), True, white)
		screen.blit(fnt, (30, 190))
		fnt = font_dot.render("ВРАЩЕНИЕ: {}".format(round(dp_rotation % 360, 2)), True, white)
		screen.blit(fnt, (30, 220))
		fnt = font_dot.render("dX: {}".format(400-int(x0)), True, white)
		screen.blit(fnt, (650, 270))
		fnt = font_dot.render("dY: {}".format(300-int(y0)), True, white)
		screen.blit(fnt, (650, 300))
		fnt = font_dot.render("ДЕЛЬТА R {:f}".format(r), True, black)
		screen.blit(fnt, (30, 560))

		if dp_scale_step == 0:
			fnt = font_dot.render("SPEED PAUSED", True, white)
			screen.blit(fnt, (30, 330))
		if disp_interlaced:
			fnt = font_dot.render("INTERLACED VIEW", True, white)
			screen.blit(fnt, (30, 360))

		fnt = font_dot.render("ИН: ЕСТЬ ССВП ГТ", True, black)
		screen.blit(fnt, (420, 560))

		# если дистанция равна 0
		if dp_scale == dp_scale_stop:
			if iis_hit:
				screen.blit( pygame.transform.scale(img_brokenglass, (800, 600)), (0,0) )
				pygame.draw.rect(screen, red, (238,280,275, 40), 2)
				pygame.draw.rect(screen, red, (243,285,265, 30))
				fnt = font_dot.render("СИГНАЛ ПОТЕРЯН", True, white)
				screen.blit(fnt, (250,285))

			else:
				pygame.draw.rect(screen, green, (218,280,330, 40), 2)
				pygame.draw.rect(screen, green, (223,285,320, 30))
				fnt = font_dot.render("УСПЕШНАЯ СТЫКОВКА", True, black)
				screen.blit(fnt, (230,285))

	# обновить главную поверхность с частотой FPS
	pygame.display.flip()
	clock.tick(fps)

pygame.quit()


from rest_framework import serializers
from taggit_serializer.serializers import TagListSerializerField, TaggitSerializer

from apps.core.models import Resource, ResourceComment

class ResourceCommentSerializer(serializers.ModelSerializer):
    class Meta:
        model = ResourceComment
        fields = [
            'id',
            'created_at',
            'last_updated_at',
            'commenter',
            'title',
            'content',
        ]
        read_only_fields = [
            'id',
            'created_at',
            'last_updated_at',
        ]

class ResourceSerializer(TaggitSerializer, serializers.ModelSerializer):
    comments = ResourceCommentSerializer(read_only=True, many=True)
    tags = TagListSerializerField()

    class Meta:
        model = Resource
        fields = [
            'id',
            'created_at',
            'last_updated_at',
            'name',
            'link',
            'creator',
            'approved',
            'comments',
            'tags',
        ]
        read_only_fields = [
            'id',
            'created_at',
            'last_updated_at',
        ]


from datetime import datetime

from pydantic import BaseModel

class TextFragment(BaseModel):
    """
    Receives a text fragment which is used for searching the documents.
    """
    text: str

    class Config:
        schema_extra = {
            'example': {
                'secret': "Text",
            }
        }

class Document(BaseModel):
    """
    The document with text from the database.
    """
    id: int
    text: str
    rubrics: list[str]
    created_date: datetime

    class Config:
        schema_extra = {
            'example': {
                'id': 1,
                'text': "Matched text",
                'rubrics': ['rubric-1', 'rubric-2'],
                'created_date': '2019-11-12T21:10:52'
            }
        }


import sys

# Try to load one of the humann modules to check the installation
try:
    from . import check
except ImportError:
    sys.exit("CRITICAL ERROR: Unable to find the HUMAnN python package." +
        " Please check your install.")

# Check the python version
check.python_version()

import argparse
import subprocess
import os
import time
import tempfile
import re
import logging

from . import config
from . import store
from . import utilities
from .search import prescreen
from .search import nucleotide
from .search import translated
from .quantify import families
from .quantify import modules

# name global logging instance
logger=logging.getLogger(__name__)

VERSION="4.0"

MAX_SIZE_DEMO_INPUT_FILE=35

def parse_arguments(args):
    """
    Parse the arguments from the user
    """
    parser = argparse.ArgumentParser(
        description= "HUMAnN : HMP Unified Metabolic Analysis Network 3\n",
        formatter_class=argparse.RawTextHelpFormatter,
        prog="humann")

    common_settings=parser.add_argument_group("[0] Common settings")

    common_settings.add_argument(
        "-i", "--input",
        help="input file of type {" +",".join(config.input_format_choices)+ "} \n[REQUIRED]",
        metavar="<input.fastq>",
        required=True)
    common_settings.add_argument(
        "-o", "--output",
        help="directory to write output files\n[REQUIRED]",
        metavar="<output>",
        required=True)
    common_settings.add_argument(
        "--threads",
        help="number of threads/processes\n[DEFAULT: " + str(config.threads) + "]",
        metavar="<" + str(config.threads) + ">",
        type=int,
        default=config.threads)
    common_settings.add_argument(
        "--version",
        action="version",
        version="%(prog)s v"+VERSION)

    workflow_refinement=parser.add_argument_group("[1] Workflow refinement")

    workflow_refinement.add_argument(
        "-r","--resume",
        help="bypass commands if the output files exist\n",
        action="store_true",
        default=config.resume)
    workflow_refinement.add_argument(
        "--bypass-nucleotide-index",
        help="bypass the nucleotide index step and run on the indexed ChocoPhlAn database\n",
        action="store_true",
        default=config.bypass_nucleotide_index)
    workflow_refinement.add_argument(
        "--bypass-nucleotide-search",
        help="bypass the nucleotide search steps\n",
        action="store_true",
        default=config.bypass_nucleotide_search)
    workflow_refinement.add_argument(
        "--bypass-prescreen",
        help="bypass the prescreen step and run on the full ChocoPhlAn database\n",
        action="store_true",
        default=config.bypass_prescreen)
    workflow_refinement.add_argument(
        "--bypass-translated-search",
        help="bypass the translated search step\n",
        action="store_true",
        default=config.bypass_translated_search)
    workflow_refinement.add_argument(
        "--taxonomic-profile",
        help="a taxonomic profile (the output file created by metaphlan)\n[DEFAULT: file will be created]",
        metavar="<taxonomic_profile.tsv>")
    workflow_refinement.add_argument(
        "--memory-use",
        help="the amount of memory to use\n[DEFAULT: " +
        config.memory_use + "]",
        default=config.memory_use,
        choices=config.memory_use_options)
    workflow_refinement.add_argument(
        "--input-format",
        help="the format of the input file\n[DEFAULT: format identified by software]",
        choices=config.input_format_choices)
    workflow_refinement.add_argument(
        "--search-mode",
        help="search for uniref50 or uniref90 gene families\n" +
        "[DEFAULT: based on translated database selected]",
        choices=[config.search_mode_uniref50, config.search_mode_uniref90],
        default=None)
    workflow_refinement.add_argument(
        "-v","--verbose",
        help="additional output is printed\n",
        action="store_true",
        default=config.verbose)

    tier1_prescreen=parser.add_argument_group("[2] Configure tier 1: prescreen")

    tier1_prescreen.add_argument(
        "--metaphlan",
        help="directory containing the MetaPhlAn software\n[DEFAULT: $PATH]",
        metavar="<metaphlan>")
    tier1_prescreen.add_argument(
        "--metaphlan-options",
        help="options to be provided to the MetaPhlAn software\n[DEFAULT: \"" + str(" ".join(config.metaphlan_opts)) + "\"]",
        metavar="<metaphlan_options>",
        default=" ".join(config.metaphlan_opts))
    tier1_prescreen.add_argument(
        "--prescreen-threshold",
        help="minimum estimated genome coverage for inclusion in pangenome search\n[DEFAULT: "
            + str(config.prescreen_threshold) + "]",
        metavar="<" + str(config.prescreen_threshold) + ">",
        type=float,
        default=config.prescreen_threshold)
    tier1_prescreen.add_argument(
        "--average-read-length",
        help="average read length for input file\n[DEFAULT: "
            + str(config.average_read_length) + "]",
        metavar="<" + str(config.average_read_length) + ">",
        type=float,
        default=config.average_read_length)

    tier2_nucleotide_search=parser.add_argument_group("[3] Configure tier 2: nucleotide search")

    tier2_nucleotide_search.add_argument(
        "--bowtie2",
        help="directory containing the bowtie2 executable\n[DEFAULT: $PATH]",
        metavar="<bowtie2>")
    tier2_nucleotide_search.add_argument(
        "--bowtie-options",
        help="options to be provided to the bowtie software\n[DEFAULT: \"" + str(" ".join(config.bowtie2_align_opts)) + "\"]",
        metavar="<bowtie_options>",
        default=" ".join(config.bowtie2_align_opts))
    tier2_nucleotide_search.add_argument(
        "--nucleotide-database",
        help="directory containing the nucleotide database\n[DEFAULT: "
            + config.nucleotide_database + "]",
        metavar="<nucleotide_database>")
    tier2_nucleotide_search.add_argument(
        "--nucleotide-identity-threshold",
        help="identity threshold for nuclotide alignments\n[DEFAULT: " + str(config.nucleotide_identity_threshold) + "]",
        metavar="<" + str(config.nucleotide_identity_threshold) + ">",
        type=float)
    tier2_nucleotide_search.add_argument(
        "--nucleotide-query-coverage-threshold",
        help="query coverage threshold for nucleotide alignments\n[DEFAULT: "
            + str(config.nucleotide_query_coverage_threshold) + "]",
        metavar="<" + str(config.nucleotide_query_coverage_threshold) + ">",
        type=float,
        default=config.nucleotide_query_coverage_threshold)
    tier2_nucleotide_search.add_argument(
        "--nucleotide-subject-coverage-threshold",
        help="subject coverage threshold for nucleotide alignments\n[DEFAULT: "
            + str(config.nucleotide_subject_coverage_threshold) + "]",
        metavar="<" + str(config.nucleotide_subject_coverage_threshold) + ">",
        type=float,
        default=config.nucleotide_subject_coverage_threshold)

    tier3_translated_search=parser.add_argument_group("[3] Configure tier 2: translated search")

    tier3_translated_search.add_argument(
        "--diamond",
        help="directory containing the diamond executable\n[DEFAULT: $PATH]",
        metavar="<diamond>")
    tier3_translated_search.add_argument(
        "--diamond-options",
        help="options to be provided to the diamond software\n[DEFAULT: \"" + str(" ".join(config.diamond_opts_uniref90)) + "\"]",
        metavar="<diamond_options>",
        default=" ".join(config.diamond_opts_uniref90))
    tier3_translated_search.add_argument(
        "--evalue",
        help="the evalue threshold to use with the translated search\n[DEFAULT: " + str(config.evalue_threshold) + "]",
        metavar="<" + str(config.evalue_threshold) + ">",
        type=float,
        default=config.evalue_threshold)
    tier3_translated_search.add_argument(
        "--protein-database",
        help="directory containing the protein database\n[DEFAULT: "
            + config.protein_database + "]",
        metavar="<protein_database>")
    tier3_translated_search.add_argument(
        "--rapsearch",
        help="directory containing the rapsearch executable\n[DEFAULT: $PATH]",
        metavar="<rapsearch>")
    tier3_translated_search.add_argument(
        "--translated-alignment",
        help="software to use for translated alignment\n[DEFAULT: " +
            config.translated_alignment_selected + "]",
        default=config.translated_alignment_selected,
        choices=config.translated_alignment_choices)
    tier3_translated_search.add_argument(
        "--translated-identity-threshold",
        help="identity threshold for translated alignments\n[DEFAULT: "
            + "Tuned automatically (based on uniref mode) unless a custom value is specified]",
        metavar="<Automatically: 50.0 or 80.0, Custom: 0.0-100.0>",
        type=float,
        dest="identity_threshold")
    tier3_translated_search.add_argument(
        "--translated-query-coverage-threshold",
        help="query coverage threshold for translated alignments\n[DEFAULT: "
            + str(config.translated_query_coverage_threshold) + "]",
        metavar="<" + str(config.translated_query_coverage_threshold) + ">",
        type=float,
        default=config.translated_query_coverage_threshold)
    tier3_translated_search.add_argument(
        "--translated-subject-coverage-threshold",
        help="subject coverage threshold for translated alignments\n[DEFAULT: "
            + str(config.translated_subject_coverage_threshold) + "]",
        metavar="<" + str(config.translated_subject_coverage_threshold) + ">",
        type=float,
        default=config.translated_subject_coverage_threshold)
    tier3_translated_search.add_argument(
        "--usearch",
        help="directory containing the usearch executable\n[DEFAULT: $PATH]",
        metavar="<usearch>")

    gene_and_pathway=parser.add_argument_group("[5] Gene and pathway quantification")

    gene_and_pathway.add_argument(
        "--count-normalization",
        help="normalization mode for results from both nucleotide and translated search\n[DEFAULT: " +
        config.count_normalization + "]",
        default=config.count_normalization,
        choices=config.count_normalization_choices)
    gene_and_pathway.add_argument(
        "--utility-database",
        help="directory containing the utility database\n[DEFAULT: "
            + config.utility_mapping_database + "]",
        metavar="<utility_mapping_database>")
    gene_and_pathway.add_argument(
        "--gap-fill",
        help="turn on/off the gap fill computation\n[DEFAULT: " +
        config.gap_fill_toggle + "]",
        default=config.gap_fill_toggle,
        choices=config.toggle_choices)
    gene_and_pathway.add_argument(
        "--minpath",
        help="turn on/off the minpath computation\n[DEFAULT: " +
        config.minpath_toggle + "]",
        default=config.minpath_toggle,
        choices=config.toggle_choices)
    gene_and_pathway.add_argument(
        "--pathways",
        help="the database to use for pathway computations\n[DEFAULT: " +
        config.pathways_database + "]",
        default=config.pathways_database,
        choices=config.pathways_database_choices)
    gene_and_pathway.add_argument(
        "--pathways-database",
        help="mapping file (or files, at most two in a comma-delimited list) to use for pathway computations\n[DEFAULT: " +
        config.pathways_database + " database ]",
        metavar=("<pathways_database.tsv>"),
        nargs=1)
    gene_and_pathway.add_argument(
        "--xipe",
        help="turn on/off the xipe computation\n[DEFAULT: " +
        config.xipe_toggle + "]",
        default=config.xipe_toggle,
        choices=config.toggle_choices)
    gene_and_pathway.add_argument(
        "--annotation-gene-index",
        help="the index of the gene in the sequence annotation\n[DEFAULT: "
            + ",".join(str(i) for i in config.chocophlan_gene_indexes) + "]",
        metavar="<"+",".join(str(i) for i in config.chocophlan_gene_indexes)+">",
        default=",".join(str(i) for i in config.chocophlan_gene_indexes))
    gene_and_pathway.add_argument(
        "--id-mapping",
        help="id mapping file for alignments\n[DEFAULT: alignment reference used]",
        metavar="<id_mapping.tsv>")

    more_output_config=parser.add_argument_group("[6] More output configuration")

    more_output_config.add_argument(
        "--remove-temp-output",
        help="remove temp output files\n" +
            "[DEFAULT: temp files are not removed]",
        action="store_true")
    more_output_config.add_argument(
        "--log-level",
        help="level of messages to display in log\n" +
        "[DEFAULT: " + config.log_level + "]",
        default=config.log_level,
        choices=config.log_level_choices)
    more_output_config.add_argument(
        "--o-log",
        help="log file\n" +
        "[DEFAULT: sample_0.log]",
        metavar="<sample_0.log>")
    more_output_config.add_argument(
        "--output-basename",
        help="the basename for the output files\n[DEFAULT: " +
        "input file basename]",
        default=config.file_basename,
        metavar="<sample_name>")
    more_output_config.add_argument(
        "--output-format",
        help="the format of the output files\n[DEFAULT: " +
        config.output_format + "]",
        default=config.output_format,
        choices=config.output_format_choices)
    more_output_config.add_argument(
        "--output-max-decimals",
        help="the number of decimals to output\n[DEFAULT: " +
        str(config.output_max_decimals) + "]",
        metavar="<" + str(config.output_max_decimals) + ">",
        type=int,
        default=config.output_max_decimals)
    more_output_config.add_argument(
        "--remove-column-description-output",
        help="remove the description in the output column\n" +
            "[DEFAULT: output column includes description]",
        action="store_true",
        default=config.remove_column_description_output)
    more_output_config.add_argument(
        "--remove-stratified-output",
        help="remove stratification from output\n" +
            "[DEFAULT: output is stratified]",
        action="store_true",
        default=config.remove_stratified_output)

    return parser.parse_args()

def update_configuration(args):
    """
    Update the configuration settings based on the arguments
    """

    # Use the full path to the input file
    args.input=os.path.abspath(args.input)

    # set the version header
    config.version_header=config.version_header+VERSION

    # If set, append paths executable locations
    if args.metaphlan:
        utilities.add_exe_to_path(os.path.abspath(args.metaphlan))

    if args.bowtie2:
        utilities.add_exe_to_path(os.path.abspath(args.bowtie2))

    if args.usearch:
        utilities.add_exe_to_path(os.path.abspath(args.usearch))

    if args.rapsearch:
        utilities.add_exe_to_path(os.path.abspath(args.rapsearch))

    if args.diamond:
        utilities.add_exe_to_path(os.path.abspath(args.diamond))

    # Set the metaphlan options, removing any extra spaces
    if args.metaphlan_options:
        config.metaphlan_opts=list(filter(None,args.metaphlan_options.split(" ")))

    # check for custom diamond options
    if args.diamond_options and args.diamond_options != " ".join(config.diamond_opts_uniref90):
        config.diamond_options_custom = True
        config.diamond_opts=list(filter(None,args.diamond_options.split(" ")))

    if args.bowtie_options:
        config.bowtie2_align_opts=list(filter(None,args.bowtie_options.split(" ")))

    # Set the pathways database selection
    if args.pathways == "metacyc":
        config.pathways_database_part1=config.metacyc_gene_to_reactions
        config.pathways_database_part2=config.metacyc_reactions_to_pathways
        config.pathways_ec_column=True
    elif args.pathways == "unipathway":
        config.pathways_database_part1=config.unipathway_database_part1
        config.pathways_database_part2=config.unipathway_database_part2
        config.pathways_ec_column=True

    # Set the locations of the pathways databases
    # If provided by the user, this will take precedence over the pathways database selection
    if args.pathways_database:
        custom_pathways_files=args.pathways_database[0].split(",")
        if len(custom_pathways_files)==2:
            config.pathways_database_part1=os.path.abspath(custom_pathways_files[0])
            config.pathways_database_part2=os.path.abspath(custom_pathways_files[1])
        elif len(custom_pathways_files)==1:
            config.pathways_database_part1=None
            config.pathways_database_part2=os.path.abspath(custom_pathways_files[0])
        else:
            sys.exit("ERROR: Please provide one or two pathways files.")
        config.pathways_ec_column=False

    # Set the locations of the other databases
    if args.nucleotide_database:
        config.nucleotide_database=os.path.abspath(args.nucleotide_database)

    if args.protein_database:
        config.protein_database=os.path.abspath(args.protein_database)

    # if set, update the config run mode to resume
    if args.resume:
        config.resume=True

    # if set, update the config run mode to verbose
    if args.verbose:
        config.verbose=True

    # if set, update the config run mode to bypass prescreen step
    if args.bypass_prescreen:
        config.bypass_prescreen=True

    # if set, update the config run mode to bypass nucleotide index step
    if args.bypass_nucleotide_index:
        config.bypass_nucleotide_index=True
        config.bypass_prescreen=True

    # if set, update the config run mode to bypass translated search step
    # set the pick_frames toggle based on the bypass
    config.pick_frames_toggle="off"
    if args.bypass_translated_search:
        config.bypass_translated_search=True

    # if set, update the config run mode to bypass nucleotide search steps
    if args.bypass_nucleotide_search:
        config.bypass_prescreen=True
        config.bypass_nucleotide_index=True
        config.bypass_nucleotide_search=True

    # update the average read length
    config.average_read_length=args.average_read_length

    # Update thresholds
    config.prescreen_threshold=args.prescreen_threshold
    config.translated_subject_coverage_threshold=args.translated_subject_coverage_threshold
    config.nucleotide_subject_coverage_threshold=args.nucleotide_subject_coverage_threshold
    config.translated_query_coverage_threshold=args.translated_query_coverage_threshold
    config.nucleotide_query_coverage_threshold=args.nucleotide_query_coverage_threshold

    # Update the max decimals output
    config.output_max_decimals=args.output_max_decimals

    # Update memory use
    config.memory_use=args.memory_use

    # Update threads
    config.threads=args.threads

    # Update the evalue threshold
    config.evalue_threshold=args.evalue

    # Update translated alignment software
    config.translated_alignment_selected=args.translated_alignment

    # Update the computation toggle choices
    config.xipe_toggle=args.xipe
    config.minpath_toggle=args.minpath
    config.gap_fill_toggle=args.gap_fill
    config.count_normalization=args.count_normalization

    if args.utility_database:
        config.utility_mapping_database=os.path.abspath(args.utility_database)

    # Check that the input file exists and is readable
    if not os.path.isfile(args.input):
        sys.exit("CRITICAL ERROR: Can not find input file selected: "+ args.input)

    if not os.access(args.input, os.R_OK):
        sys.exit("CRITICAL ERROR: Not able to read input file selected: " + args.input)

    # Update the output format
    config.remove_stratified_output=args.remove_stratified_output
    config.remove_column_description_output=args.remove_column_description_output

    # Check that the output directory is writeable
    output_dir = os.path.abspath(args.output)

    if not os.path.isdir(output_dir):
        try:
            print("Creating output directory: " + output_dir)
            os.mkdir(output_dir)
        except EnvironmentError:
            sys.exit("CRITICAL ERROR: Unable to create output directory.")

    if not os.access(output_dir, os.W_OK):
        sys.exit("CRITICAL ERROR: The output directory is not " +
            "writeable. This software needs to write files to this directory.\n" +
            "Please select another directory.")

    print("Output files will be written to: " + output_dir)

    # Set the basename of the output files if specified as an option
    if args.output_basename:
        config.file_basename=args.output_basename
    else:
        # Determine the basename of the input file to use as output file basename
        input_file_basename=os.path.basename(args.input)
        # Remove gzip extension if present
        if re.search('.gz$',input_file_basename):
            input_file_basename='.'.join(input_file_basename.split('.')[:-1])
        # Remove input file extension if present
        if '.' in input_file_basename:
            input_file_basename='.'.join(input_file_basename.split('.')[:-1])

        config.file_basename=input_file_basename

    # Set the output format
    config.output_format=args.output_format

    # Set final output file names and location
    config.pathabundance_file=os.path.join(output_dir,
            config.file_basename + config.pathabundance_file + "." +
            config.output_format)
    config.pathcoverage_file=os.path.join(output_dir,
            config.file_basename + config.pathcoverage_file + "." +
            config.output_format)
    config.genefamilies_file=os.path.join(output_dir,
            config.file_basename + config.genefamilies_file + "." +
            config.output_format)
    config.profile_file=os.path.join(output_dir,
            config.file_basename + config.profile_file + "." +
            config.output_format)
    config.reactions_file=os.path.join(output_dir,
            config.file_basename + config.reactions_file + "." +
            config.output_format)

    # set the location of the temp directory
    if not args.remove_temp_output:
        config.temp_dir=os.path.join(output_dir,config.file_basename+"_humann_temp")
        if not os.path.isdir(config.temp_dir):
            try:
                os.mkdir(config.temp_dir)
            except EnvironmentError:
                sys.exit("Unable to create temp directory: " + config.temp_dir)
    else:
        config.temp_dir=tempfile.mkdtemp(
            prefix=config.file_basename+'_humann_temp_',dir=output_dir)

    # create the unnamed temp directory
    config.unnamed_temp_dir=tempfile.mkdtemp(dir=config.temp_dir)

    # set the name of the log file
    log_file=os.path.join(output_dir,config.file_basename+"_0.log")

    # change file name if set
    if args.o_log:
        log_file=args.o_log

    # configure the logger
    logging.basicConfig(filename=log_file,format='%(asctime)s - %(name)s - %(levelname)s: %(message)s',
        level=getattr(logging,args.log_level), filemode='w', datefmt='%m/%d/%Y %I:%M:%S %p')

    # write the version of the software to the log
    logger.info("Running humann v"+VERSION)

    # write the location of the output files to the log
    logger.info("Output files will be written to: " + output_dir)

    # write the location of the temp file directory to the log
    message="Writing temp files to directory: " + config.temp_dir
    logger.info(message)
    if config.verbose:
        print("\n"+message+"\n")

    return log_file

def parse_chocophlan_gene_indexes(annotation_gene_index):
    """ Parse the chocophlan gene index input """

    # Update the chocophlan gene indexes
    chocophlan_gene_indexes=[]
    for index in annotation_gene_index.split(","):
        # Look for array range
        if ":" in index:
            split_index=index.split(":")
            start=split_index[0]
            end=split_index[1]

            try:
                start=int(start)
                end=int(end)
                chocophlan_gene_indexes+=range(start,end)
            except ValueError:
                pass
        else:
            # Convert to int
            try:
                index=int(index)
                chocophlan_gene_indexes.append(index)
            except ValueError:
                pass

    return chocophlan_gene_indexes

def check_requirements(args):
    """
    Check requirements (file format, dependencies, permissions)
    """

    # Check the pathways database files exist and are readable
    if config.pathways_database_part1:
        utilities.file_exists_readable(config.pathways_database_part1)
    utilities.file_exists_readable(config.pathways_database_part2)

    # Determine the input file format if not provided
    if not args.input_format:
        args.input_format=utilities.determine_file_format(args.input)

        if args.input_format == "unknown":
            sys.exit("CRITICAL ERROR: Unable to determine the input file format." +
                " Please provide the format with the --input_format argument.")

    config.input_format = args.input_format

    # If the input file is compressed, then decompress
    if args.input_format.endswith(".gz"):
        new_file=utilities.gunzip_file(args.input)

        if new_file:
            args.input=new_file
            args.input_format=args.input_format.split(".")[0]
        else:
            sys.exit("CRITICAL ERROR: Unable to use gzipped input file. " +
                " Please check the format of the input file.")

    # check if the input file has sequence identifiers of the new illumina casava v1.8+ format
    # these have spaces causing the paired end reads to have the same identifier after
    # delimiting by space (causing bowtie2/diamond to label the reads with the same identifier)
    if args.input_format in ["fasta","fastq"]:
        if utilities.space_in_identifier(args.input):
            message="Removing spaces from identifiers in input file"
            print(message+" ...\n")
            logger.info(message)
            new_file = utilities.remove_spaces_from_file(args.input)

            if new_file:
                args.input=new_file
            else:
                sys.exit("CRITICAL ERROR: Unable to remove spaces from identifiers in input file.")

    # If the input format is in binary then convert to sam (tab-delimited text)
    if args.input_format == "bam":

        # Check for the samtools software
        if not utilities.find_exe_in_path("samtools"):
            sys.exit("CRITICAL ERROR: The samtools executable can not be found. "
            "Please check the install or select another input format.")

        new_file=utilities.bam_to_sam(args.input)

        if new_file:
            args.input=new_file
            args.input_format="sam"
        else:
            sys.exit("CRITICAL ERROR: Unable to convert bam input file to sam.")

    # If the input format is in biom then convert to tsv
    if args.input_format == "biom":

        # Check for the biom software
        if not utilities.find_exe_in_path("biom"):
            sys.exit("CRITICAL ERROR: The biom executable can not be found. "
            "Please check the install or select another input format.")

        new_file=utilities.biom_to_tsv(args.input)

        if new_file:
            args.input=new_file
            # determine the format of the file
            args.input_format=utilities.determine_file_format(args.input)
        else:
            sys.exit("CRITICAL ERROR: Unable to convert biom input file to tsv.")

    # If the biom output format is selected, check for the biom package
    if config.output_format=="biom":
        if not utilities.find_exe_in_path("biom"):
            sys.exit("CRITICAL ERROR: The biom executable can not be found. "
            "Please check the install or select another output format.")

        try:
            import biom
        except ImportError:
            sys.exit("Could not find the biom software."+
                " This software is required since the output file is a biom file.")

    if os.path.basename(config.utility_mapping_database) == "utility_DEMO":
        # Check the input file is a demo input if running with demo database
        try:
            input_file_size=os.path.getsize(args.input)/1024**2
        except EnvironmentError:
            input_file_size=0
        if input_file_size > MAX_SIZE_DEMO_INPUT_FILE:
            sys.exit("ERROR: You are using the demo utility database with "
                + "a non-demo input file. If you have not already done so, please "
                + "run humann_databases to download the full utility database. "
                + "If you have downloaded the full database, use the option "
                + "--utility-database to provide the location. "
                + "You can also run humann_config to update the default "
                + "database location. For additional information, please "
                + "see the HUMAnN User Manual.")

    # If the file is fasta/fastq check for requirements
    if args.input_format in ["fasta","fastq"]:
        # Check that the chocophlan directory exists
        if not config.bypass_nucleotide_index:
            if not os.path.isdir(config.nucleotide_database):
                if args.nucleotide_database:
                    sys.exit("CRITICAL ERROR: The directory provided for the ChocoPhlAn database at "
                        + args.nucleotide_database + " does not exist. Please select another directory.")
                else:
                    sys.exit("CRITICAL ERROR: The default ChocoPhlAn database directory of "
                        + config.nucleotide_database + " does not exist. Please provide the location "
                        + "of the ChocoPhlAn directory using the --nucleotide-database option.")

        # Check that the files in the chocophlan folder are of the right format
        if not config.bypass_nucleotide_index:
            valid_format_count=0
            for file in os.listdir(config.nucleotide_database):
                # expect most of the file names to be of the format g__*s__*
                if re.search("^SGB",file):
                    valid_format_count+=1

                if not config.metaphlan_v4_db_matching_uniref in file:
                    sys.exit("\n\nCRITICAL ERROR: The directory provided for ChocoPhlAn contains files ( "+file+" )"+\
                        " that are not of the expected version. Please install the latest version"+\
                        " of the database: "+config.metaphlan_v4_db_matching_uniref)

            if valid_format_count == 0:
                sys.exit("CRITICAL ERROR: The directory provided for ChocoPhlAn does not "
                    + "contain files of the expected format (ie \'^SGB\').")

        # Check if running with the demo database
        if not config.bypass_nucleotide_index:
            if os.path.basename(config.nucleotide_database) == "chocophlan_DEMO":
                # Check the input file is a demo input if running with demo database
                try:
                    input_file_size=os.path.getsize(args.input)/1024**2
                except EnvironmentError:
                    input_file_size=0
                if input_file_size > MAX_SIZE_DEMO_INPUT_FILE:
                    sys.exit("ERROR: You are using the demo ChocoPhlAn database with "
                        + "a non-demo input file. If you have not already done so, please "
                        + "run humann_databases to download the full ChocoPhlAn database. "
                        + "If you have downloaded the full database, use the option "
                        + "--nucleotide-database to provide the location. "
                        + "You can also run humann_config to update the default "
                        + "database location. For additional information, please "
                        + "see the HUMAnN User Manual.")

        # Check that the metaphlan2 executable can be found
        if not config.bypass_prescreen and not config.bypass_nucleotide_index:
            if not utilities.find_exe_in_path("metaphlan"):
                sys.exit("CRITICAL ERROR: The metaphlan executable can not be found. "
                    "Please check the install.")

            # Check the metaphlan2 version
            utilities.check_software_version("metaphlan",config.metaphlan_version)

        # Check that the bowtie2 executable can be found
        if not config.bypass_nucleotide_search:
            if not utilities.find_exe_in_path("bowtie2"):
                sys.exit("CRITICAL ERROR: The bowtie2 executable can not be found. "
                    "Please check the install.")

            # Check the bowtie2 version
            utilities.check_software_version("bowtie2", config.bowtie2_version, warning=True)

        if not config.bypass_translated_search:
            # Check that the protein database directory exists
            if not os.path.isdir(config.protein_database):
                if args.protein_database:
                    sys.exit("CRITICAL ERROR: The directory provided for the protein database at "
                        + args.protein_database + " does not exist. Please select another directory.")
                else:
                    sys.exit("CRITICAL ERROR: The default protein database directory of "
                        + config.protein_database + " does not exist. Please provide the location "
                        + "of the directory using the --protein-database option.")

            # Check that some files in the protein database folder are of the expected extension
            expected_database_extension=""
            if config.translated_alignment_selected == "usearch":
                expected_database_extension=config.usearch_database_extension
            elif config.translated_alignment_selected == "rapsearch":
                expected_database_extension=config.rapsearch_database_extension
            elif config.translated_alignment_selected == "diamond":
                expected_database_extension=config.diamond_database_extension

            valid_format_count=0
            database_files=os.listdir(config.protein_database)
            valid_format_database_files=[]
            for file in database_files:
                if not config.matching_uniref in file:
                    sys.exit("\n\nCRITICAL ERROR: The directory provided for the translated database contains files ( "+file+" )"+\
                        " that are not of the expected version. Please install the latest version"+\
                        " of the database: "+config.matching_uniref)

                if file.endswith(expected_database_extension):
                    # if rapsearch check for the second database file
                    if config.translated_alignment_selected == "rapsearch":
                        database_file=re.sub(config.rapsearch_database_extension+"$","",file)
                        if database_file in database_files:
                            valid_format_count+=1
                            valid_format_database_files.append(database_file)
                    else:
                        valid_format_count+=1
                        valid_format_database_files.append(file)

            if valid_format_count == 0:
                sys.exit("CRITICAL ERROR: The protein database directory provided ( " + config.protein_database
                    + " ) does not contain any files that have been formatted to run with"
                    " the translated alignment software selected ( " +
                    config.translated_alignment_selected + " ). Please format these files so"
                    + " they are of the expected extension ( " + expected_database_extension +" ).")

            # if a search mode is not set by the user, then try to get mode based on translated search database
            if not args.search_mode:
                if config.search_mode_uniref90 in valid_format_database_files[0].lower():
                    logger.info("Search mode set to uniref90 because a uniref90 translated search database is selected")
                    config.search_mode=config.search_mode_uniref90
                elif config.search_mode_uniref50 in valid_format_database_files[0].lower():
                    logger.info("Search mode set to uniref50 because a uniref50 translated search database is selected")
                    config.search_mode=config.search_mode_uniref50

        # Check if running with the demo database
        if not config.bypass_translated_search:
            if os.path.basename(config.protein_database) == "uniref_DEMO":
                # Check the input file is a demo input if running with demo database
                try:
                    input_file_size=os.path.getsize(args.input)/1024**2
                except EnvironmentError:
                    input_file_size=0
                if input_file_size > MAX_SIZE_DEMO_INPUT_FILE:
                    sys.exit("ERROR: You are using the demo UniRef database with "
                        + "a non-demo input file. If you have not already done so, please "
                        + "run humann_databases to download the full UniRef database. "
                        + "If you have downloaded the full database, use the option "
                        + "--protein-database to provide the location. "
                        + "You can also run humann_config to update the default "
                        + "database location. For additional information, please "
                        + "see the HUMAnN User Manual.")

            # Check that the translated alignment executable can be found
            if not utilities.find_exe_in_path(config.translated_alignment_selected):
                sys.exit("CRITICAL ERROR: The " +  config.translated_alignment_selected +
                    " executable can not be found. Please check the install.")

            # Check for correct usearch version
            if config.translated_alignment_selected == "usearch":
                utilities.check_software_version("usearch",config.usearch_version)

            # Check for correct rapsearch version
            if config.translated_alignment_selected == "rapsearch":
                utilities.check_software_version("rapsearch", config.rapsearch_version)

            # Check for the correct diamond version
            if config.translated_alignment_selected == "diamond":
                utilities.check_software_version("diamond", config.diamond_version)

    # parse the chocophlan gene index
    chocophlan_gene_indexes=parse_chocophlan_gene_indexes(args.annotation_gene_index)

    # set the user provided search mode, if set
    if args.search_mode:
        config.search_mode = args.search_mode

    # set the values based on the search mode
    if config.search_mode == config.search_mode_uniref90:
        # only change identity threshold to default if user has not provided a specific setting
        if args.identity_threshold is None:
            config.identity_threshold = config.identity_threshold_uniref90_mode
        else:
            config.identity_threshold = args.identity_threshold

        # only change chocophlan gene index if user has not provided a specific setting
        if config.chocophlan_gene_indexes == chocophlan_gene_indexes:
            config.chocophlan_gene_indexes = config.chocophlan_gene_indexes_uniref90_mode
        else:
            config.chocophlan_gene_indexes = chocophlan_gene_indexes

        # set the diamond options if custom options were not provided
        if not config.diamond_options_custom:
            config.diamond_opts = config.diamond_opts_uniref90
    else:
        # only change identity threshold to default if user has not provided a specific setting
        if args.identity_threshold is None:
            config.identity_threshold = config.identity_threshold_uniref50_mode
        else:
            config.identity_threshold = args.identity_threshold

        # only change chocophlan gene index if user has not provided a specific setting
        if config.chocophlan_gene_indexes == chocophlan_gene_indexes:
            config.chocophlan_gene_indexes = config.chocophlan_gene_indexes_uniref50_mode
        else:
            config.chocophlan_gene_indexes = chocophlan_gene_indexes

        # set the diamond options if custom options were not provided
        if not config.diamond_options_custom:
            config.diamond_opts = config.diamond_opts_uniref50

def timestamp_message(task, start_time):
    """
    Print and log a message about the task completed and the time
    Log messages are tab delimited for quick task/time access with awk
    Return the new start time
    """
    message="TIMESTAMP: Completed \t" + task + " \t:\t " + \
        str(int(round(time.time() - start_time))) + "\t seconds"
    logger.info(message)
    if config.verbose:
        print("\n"+message.replace("\t","")+"\n")

    return time.time()

def main():
    # Parse arguments from command line
    args=parse_arguments(sys.argv)

    # Update the configuration settings based on the arguments
    log_file=update_configuration(args)

    # Check for required files, software, databases, and also permissions
    check_requirements(args)

    # Write the config settings to the log file
    config.log_settings()

    # Initialize alignments and gene scores
    minimize_memory_use=True
    if config.memory_use == "maximum":
        minimize_memory_use=False

    alignments=store.Alignments(minimize_memory_use=minimize_memory_use)
    unaligned_reads_store=store.Reads(minimize_memory_use=minimize_memory_use)
    gene_scores=store.GeneScores()

    # If id mapping is provided then process
    if args.id_mapping:
        alignments.process_id_mapping(args.id_mapping)

    # Load in the reactions database
    reactions_database=None
    if config.pathways_database_part1:
        reactions_database=store.ReactionsDatabase(config.pathways_database_part1)

        message="Load pathways database part 1: " + config.pathways_database_part1
        logger.info(message)

    # Load in the pathways database
    pathways_database=store.PathwaysDatabase(config.pathways_database_part2, reactions_database)

    if config.pathways_database_part1:
        message="Load pathways database part 2: " + config.pathways_database_part2
    else:
        message="Load pathways database: " + config.pathways_database_part2
    logger.info(message)

    # Start timer
    start_time=time.time()

    # Process fasta or fastq input files
    output_files=[]
    if args.input_format in ["fasta","fastq"]:
        # Run prescreen to identify bugs
        bug_file = "Empty"
        if args.taxonomic_profile:
            bug_file = os.path.abspath(args.taxonomic_profile)
        else:
            if not config.bypass_prescreen:
                bug_file = prescreen.alignment(args.input)
                output_files.append(bug_file)
                start_time=timestamp_message("prescreen",start_time)

        # Create the custom database from the bugs list
        custom_database = ""
        if not config.bypass_nucleotide_index:
            custom_database = prescreen.create_custom_database(config.nucleotide_database, bug_file)
            start_time=timestamp_message("custom database creation",start_time)
        else:
            custom_database = "Bypass"

        # Run nucleotide search on custom database
        if custom_database != "Empty" and not config.bypass_nucleotide_search:
            if not config.bypass_nucleotide_index:
                nucleotide_index_file = nucleotide.index(custom_database)
                start_time=timestamp_message("database index",start_time)
            else:
                nucleotide_index_file = nucleotide.find_index(config.nucleotide_database)

            nucleotide_alignment_file = nucleotide.alignment(args.input,
                nucleotide_index_file)

            start_time=timestamp_message("nucleotide alignment",start_time)

            # Determine which reads are unaligned and reduce aligned reads file
            # Remove the alignment_file as we only need the reduced aligned reads file
            [ unaligned_reads_file_fasta, reduced_aligned_reads_file ] = nucleotide.unaligned_reads(
                nucleotide_alignment_file, alignments, unaligned_reads_store, keep_sam=True)

            start_time=timestamp_message("nucleotide alignment post-processing",start_time)

            # Print out total alignments per bug
            message="Total bugs from nucleotide alignment: " + str(alignments.count_bugs())
            logger.info(message)
            print(message)

            message=alignments.counts_by_bug()
            logger.info("\n"+message)
            print(message)

            message="Total gene families from nucleotide alignment: " + str(alignments.count_genes())
            logger.info(message)
            print("\n"+message)

            # Report reads unaligned
            message="Unaligned reads after nucleotide alignment: " + utilities.estimate_unaligned_reads_stored(
                args.input, unaligned_reads_store) + " %"
            logger.info(message)
            print("\n"+message+"\n")
        else:
            logger.debug("Custom database is empty")
            reduced_aligned_reads_file = "Empty"
            unaligned_reads_file_fasta=args.input
            unaligned_reads_store=store.Reads(unaligned_reads_file_fasta, minimize_memory_use=minimize_memory_use)

        # Do not run if set to bypass translated search in config file
        if not config.bypass_translated_search:
            # Run translated search on UniRef database if unaligned reads exit
            if unaligned_reads_store.count_reads()>0:
                translated_alignment_file = translated.alignment(config.protein_database,
                    unaligned_reads_file_fasta)

                start_time=timestamp_message("translated alignment",start_time)

                # Determine which reads are unaligned
                translated_unaligned_reads_file_fastq = translated.unaligned_reads(
                    unaligned_reads_store, translated_alignment_file, alignments)

                start_time=timestamp_message("translated alignment post-processing",start_time)

                # Print out total alignments per bug
                message="Total bugs after translated alignment: " + str(alignments.count_bugs())
                logger.info(message)
                print(message)

                message=alignments.counts_by_bug()
                logger.info("\n"+message)
                print(message)

                message="Total gene families after translated alignment: " + str(alignments.count_genes())
                logger.info(message)
                print("\n"+message)

                # Report reads unaligned
                message="Unaligned reads after translated alignment: " + utilities.estimate_unaligned_reads_stored(
                    args.input, unaligned_reads_store) + " %"
                logger.info(message)
                print("\n"+message+"\n")
            else:
                message="All reads are aligned so translated alignment will not be run"
                logger.info(message)
                print(message)
        else:
            message="Bypass translated search"
            logger.info(message)
            print(message)

    # Process input files of sam format
    elif args.input_format in ["sam"]:

        # Store the sam mapping results
        message="Process the sam mapping results ..."
        logger.info(message)
        print("\n"+message)

        [unaligned_reads_file_fasta, reduced_aligned_reads_file] = nucleotide.unaligned_reads(
            args.input, alignments, unaligned_reads_store, keep_sam=True)

        start_time=timestamp_message("alignment post-processing",start_time)

    # Process input files of tab-delimited blast format
    elif args.input_format in ["blastm8"]:

        # Store the blastm8 mapping results
        message="Process the blastm8 mapping results ..."
        logger.info(message)
        print("\n"+message)

        translated_unaligned_reads_file_fastq = translated.unaligned_reads(
            unaligned_reads_store, args.input, alignments)

        start_time=timestamp_message("alignment post-processing",start_time)

    # Get the number of remaining unaligned reads
    unaligned_reads_count=unaligned_reads_store.count_reads()

    # Clear all of the unaligned reads as they are no longer needed
    unaligned_reads_store.clear()

    # Compute or load in gene families
    if args.input_format in ["fasta","fastq","sam","blastm8"]:
        # Compute the gene families
        message="Computing gene families ..."
        logger.info(message)
        print("\n"+message)

        families_file=families.gene_families(alignments,gene_scores,unaligned_reads_count)
        output_files.append(families_file)

        start_time=timestamp_message("computing gene families",start_time)

    elif args.input_format in ["genetable"]:
        # Load the gene scores
        message="Process the gene table ..."
        logger.info(message)
        print("\n"+message)

        unaligned_reads_count=gene_scores.add_from_file(args.input,id_mapping_file=args.id_mapping)

        start_time=timestamp_message("processing gene table",start_time)

    # Handle input files of unknown formats
    else:
        sys.exit("CRITICAL ERROR: Input file of unknown format.")

    # Clear all of the alignments data as they are no longer needed
    alignments.clear()

    # Identify reactions and then pathways from the alignments
    message="Computing reaction and pathway abundance ..."
    logger.info(message)
    print("\n"+message)
    pathways_and_reactions_store=modules.identify_reactions_and_pathways(
        gene_scores, reactions_database, pathways_database, unaligned_reads_count)

    # Compute pathway abundance and coverage
    abundance_file, coverage_file, reaction_file=modules.compute_pathways_abundance_and_coverage(
        gene_scores, reactions_database, pathways_and_reactions_store, pathways_database, unaligned_reads_count)
    output_files.append(reaction_file)
    output_files.append(abundance_file)
    output_files.append(log_file)
    #output_files.append(coverage_file)

    start_time=timestamp_message("computing pathways",start_time)

    message="\nOutput files created: \n" + "\n".join(output_files) + "\n"
    logger.info(message)
    print(message)

    # Remove the unnamed temp files
    utilities.remove_directory(config.unnamed_temp_dir)

    # Remove named temp directory
    if args.remove_temp_output:
        utilities.remove_directory(config.temp_dir)


#!/usr/bin/env python
from github import Github
import argparse
import sys
import os
import time
import random
import math

class Collect():
	def __init__(self):
		self.user=''
		self.pwd=''
		self.token=''
		self.org=''
		self.repo=''
		self.branch=''
		self.event_type=''
		self.per_page=100
		self.client=''
		self.organization=''
		self.state=''
		self.sleep_time_length = 1 #5
		self.sleep_time_range = 1 #3

	def get_arguments(self):
		parser = argparse.ArgumentParser(description="Github data collection with PyGithub")
		parser.add_argument('--user', type=str, help="The name of the user")
		parser.add_argument('--pwd', type=str,  help="Password of the user")
		parser.add_argument('--token', type=str, help="The private token of the user")
		parser.add_argument('--org', type=str,  help="The name of the Organization")
		parser.add_argument('--repo', type=str,  help="The name of the Repositories")
		parser.add_argument('--branch',  type=str, help="The name of the branches")
		parser.add_argument('--state',  type=str, help="The State of the issues")
		parser.add_argument('--event_type', type=str, help="The type of the events(e.g issues, commits")
		parser.add_argument('--per_page', default=self.per_page,  type=int, help="The number of per_page requests")
		return parser.parse_args()

	def validate_arguments(self, args):
		"""Validate arguments entered by user"""
		if args.org == None:
			print('Please specify Organization name. Exiting.')
			sys.exit(0)
		if args.repo == None:
			print('Please specify Repositories name. Exiting.')
			sys.exit(0)
		if args.event_type == None:
			print('Please specify type of the event. Exiting.')
			sys.exit(0)
		if args.event_type == 'commits' and args.branch == None:
			print('Please specify branch name. Exiting.')
			sys.exit(0)
		if args.event_type == 'issues' and args.state == None:
			print('Please specify state of the issues. Exiting.')
			sys.exit(0)
		if args.event_type == 'pullRequests' and args.branch == None and args.state == None:
			print('Please specify branch and state of the pulls. Exiting.')
			sys.exit(0)

		return

	def create_github_instance(self, args):
		if args.token != None:
			#client = Github(args.token, per_page=args.per_page)
			#self.client = Github(args.user, args.pwd, per_page=args.per_page)
			self.client = Github(args.user, login_or_token=args.token, per_page=args.per_page)

		elif args.user != None and args.pwd != None:
			self.client = Github(args.user, args.pwd, per_page=args.per_page)

		else:
			self.client = Github()

		self.organization = self.client.get_organization(args.org)

	def create_directory(self,args):
		"""Create a directory for organization, for each repository in organization, for each event type in repository"""
		# 1... create a directory for this organization
		if not os.path.exists(args.org):
			os.mkdir(args.org)
		# 2... create directory for each repository of an organization
		# get repository

		repo_list = []
		if (args.repo == 'all'):
			repo_list = [repo.name for repo in self.organization.get_repos()]
		else:
			repo_list = [args.repo]

		# create direcory
		for j in repo_list:
			if not os.path.exists(args.org + "/" + j):
				os.mkdir(args.org + "/" + j)

		# 3.... create directory for each event type of repository
		for repo_name in repo_list:
			if args.event_type == 'commits':
				branch_name_list = self.get_branch(repo_name, args)
				for branch_name in branch_name_list:
					if not os.path.exists(args.org+"/"+repo_name+"/"+args.event_type + "/"+ branch_name + "_branch"):
						os.mkdir(args.org+"/"+repo_name+"/"+args.event_type + "/"+ branch_name + "_branch")
			else:
				if not os.path.exists(args.org+"/"+repo_name+"/"+args.event_type):
					os.mkdir(args.org+"/"+repo_name+"/"+args.event_type)

	def identify_event(self,args):
		"""identify the type of event given by the user"""
		if(args.event_type == 'issues'):
			self.collect_issues(args)
		if(args.event_type == 'issueComments'):
			self.collect_issues_comments(args)
		if (args.event_type == 'issueEvents'):
			self.collect_issues_events(args)
		if (args.event_type == 'commits'):
			self.collect_commits(args)
		if (args.event_type == 'events'):
			self.collect_events(args)
		if (args.event_type == 'commitComments'):
			self.collect_commitsComments(args)
		if(args.event_type == 'pullRequests'):
			self.collect_pullRequests(args)
		if (args.event_type == 'pullRequestComments'):
			self.collect_pullRequestComments(args)

	def get_repo(self,args):
		"""store all repository in a given organization as repo_list"""
		repo_list=[]
		if(args.repo == 'all'):
			repo_list = [repo.name for repo in self.organization.get_repos()]
		else:
			repo_list = [args.repo]

		return repo_list

	def get_branch(self, repo_name, args):
		branch_list = []
		if(args.branch == 'all'):
			branch_list = [branch.name for branch in self.organization.get_repo(repo_name).get_branches()]

		else:
			branch_list = [args.branch]

		return branch_list

	def collect_issues_events(self, args):
		repo_list = self.get_repo(args)
		print("\nRepositories:\n ", repo_list)
		try:
			for repo_name in repo_list:
				print("\n\t" + repo_name + " Repository")
				repo = self.organization.get_repo(repo_name)

				totalIssueEvents = repo.get_issues_events().totalCount
				print("total number of issue events in ", repo_name, "is", totalIssueEvents)

				totalPage = totalIssueEvents / args.per_page
				if totalPage is not int:
					totalPage = math.ceil(totalPage)
				print("total number of page with per_page ", self.per_page, " is ", totalPage)

				page = 0
				num_of_issues_events = 0

				while page < totalPage:
					issues_events_list = []
					print("\n\tpage: ", page)
					for event in repo.get_issues_events().get_page(page):
						event_dict = {}
						event_dict['actor'] = event.actor
						event_dict['commit_id'] = event.commit_id
						event_dict['created_at'] = event.created_at
						event_dict['event'] = event.event
						event_dict['id'] = event.id
						event_dict['issue'] = event.issue
						event_dict['url'] = event.url
						event_dict['node_id'] = event.node_id
						event_dict['commit_url'] = event.commit_url
						event_dict['label'] = event.label
						event_dict['assignee'] = event.assignee
						issues_events_list.append(event_dict)

						num_of_issues_events += 1
						print(num_of_issues_events)

					with open(args.org + "/" + repo_name + "/" + args.event_type + "/" + args.org + "-" + repo_name + "-" +
							  args.event_type + "-page-" + str(page) + ".json", 'w') as f:
							f.write(str(issues_events_list))

					print("page ", page, " added to file")
					self.sleeper()
					page += 1

			print("data successfully collected")
		except Exception as e:
			print("Problem Occured: ", e)

	def collect_pullRequests(self, args):
		# call a get_repo function
		repo_list = self.get_repo(args)
		print("\n\tRepositories\n", repo_list)
		try:
			for repo_name in repo_list:
				print("\n", repo_name, " Repository")
				repo = self.organization.get_repo(repo_name)
				branch_list = self.get_branch(repo_name, args)
				print("branches: ", branch_list)
				for branch in branch_list:

				    total_pulls = repo.get_pulls(state=args.state, sort='created', base=branch).totalCount
				    print("total number of pulls in ", repo_name, " of branch ", branch, " is ", total_pulls)

				    total_page = total_pulls / args.per_page
				    if total_page is not int:total_page = math.ceil(total_page)
				    print("The total number of page is ", total_page)
				    page = 0
				    num_of_pulls = 0
				    while page < total_page:
				    	print("\n\tPage: ", page)
				    	pull_list=[]
				    	for pull in repo.get_pulls(state=args.state, sort='created', base=branch).get_page(page):
				    		pull_dict = {}
				    		pull_dict['id'] = pull.id
				    		pull_dict['number'] = pull.number
				    		pull_dict['title'] = pull.title
				    		pull_dict['user'] = pull.user
				    		pull_dict['body'] = pull.body
				    		pull_dict['changed_files'] = pull.changed_files
				    		pull_dict['closed_at'] = pull.closed_at
				    		pull_dict['comments'] = pull.comments
				    		pull_dict['commits'] = pull.commits
				    		pull_dict['created_at'] = pull.created_at
				    		pull_dict['commits_url'] = pull.commits_url
				    		pull_dict['review_comments'] = pull.review_comments
				    		pull_dict['head'] = pull.head
				    		pull_dict['merged'] = pull.merged
				    		pull_dict['merged_at'] = pull.merged_at
				    		pull_dict['merged_by'] = pull.merged_by
				    		pull_dict['milestone'] = pull.milestone
				    		pull_dict['merge_commit_sha'] = pull.merge_commit_sha
				    		pull_dict['mergeable'] = pull.mergeable
				    		pull_dict['mergeable_state'] = pull.mergeable_state
				    		pull_dict['labels'] = pull.labels
				    		pull_dict['deletions'] = pull.deletions
				    		pull_dict['additions'] = pull.additions
				    		pull_dict['commit_url'] = pull.commits_url
				    		pull_dict['assignee'] = pull.assignee
				    		pull_dict['assignees'] = pull.assignees
				    		pull_dict['base'] = pull.base
				    		pull_list.append(pull_dict)
				    		num_of_pulls += 1
				    		print(num_of_pulls)
				    	with open(args.org + "/" + repo_name + "/" + args.event_type + "/" + args.org + "-" + repo_name + "-" +
								  branch + "-" + args.state + "-" + args.event_type + ".json", 'w') as f:
				    		f.write(str(pull_list))
				    	self.sleeper()
				    	page += 1
			print("data successfully collected")
		except Exception as e:
			print("Problem Occured: ", e)

	def collect_pullRequestComments(self,args):
	# call a get_repo function
		repo_list = self.get_repo(args)
		print("\n\tRepositories\n", repo_list)
		try:
			for repo_name in repo_list:
				print("\n", repo_name, "Repository")
				repo = self.organization.get_repo(repo_name)
				pull_list = []
				num_of_pulls = 0
				print("total pull Request comments in ", repo_name, " is: ", repo.get_pulls_comments().totalCount)
				for pull in repo.get_pulls_comments():
				    pull_dict = {}
				    pull_dict['id'] = pull.id
				    pull_dict['user'] = pull.user
				    pull_dict['body'] = pull.body
				    pull_dict['created_at'] = pull.created_at
				    pull_dict['commit_id'] = pull.commit_id
				    pull_dict['created_at'] = pull.created_at
				    pull_dict['diff_hunk'] = pull.diff_hunk
				    pull_dict['in_reply_to_id'] = pull.in_reply_to_id
				    pull_dict['original_commit_id'] = pull.original_commit_id
				    pull_dict['original_position'] = pull.original_position
				    pull_dict['path'] = pull.path
				    pull_dict['url'] = pull.url
				    pull_dict['updated_at'] = pull.updated_at
				    pull_dict['pull_request_url'] = pull.pull_request_url

				    pull_list.append(pull_dict)
				    num_of_pulls += 1
				    print(num_of_pulls)

				with open(args.org + "/" + repo_name + "/" + args.event_type + "/" + args.org + "-" + repo_name + "-" +
				          args.event_type + ".json", 'w') as f:
					f.write(str(pull_list))

			print("data successfully collected")
		except Exception as e:
			print("Problem Occured: ", e)

	def collect_commits(self, args):

		"""collect the data of the issue event in a given repository may be all repository or one repository"""
		# call a get_repo function
		repo_list = self.get_repo(args)
		print("\n\tRepositories:\n ", repo_list)
		try:
			for repo_name in repo_list:
				print(repo_name, "Repository")
				repo = self.organization.get_repo(repo_name)
				branch_list = self.get_branch(repo_name, args)
				print("branches: ", branch_list)

				for branch in branch_list:
					git_branch = self.organization.get_repo(repo_name).get_branch(branch)
					branch_commit = git_branch.commit

					total_commits = repo.get_commits(sha=branch_commit.sha).totalCount
					print("total number of commits in ",repo_name," of branch ", branch, " is: ", total_commits)

					# since there are 100 commits in a single page we can easily get the total number of page by dividing the total commits with 100
					total_page =  total_commits / args.per_page
					if total_page is not int:
						total_page = math.ceil(total_page)
					print("The total number of page is: " + str(total_page))

					#print(repo.get_commits().get_page(rel='last'))
					page = 0
					num_of_commits = 0
					while page < total_page:#just for testing but actually its till last page
						commit_list = []
						print("\n\tpage: ", page)

						for commit in repo.get_commits(sha=branch_commit.sha).get_page(page):
							commit_dict = {}
							commit_dict['author'] = commit.author
							commit_dict['sha'] = commit.sha
							commit_dict['files'] = commit.files
							commit_dict['stats'] = commit.stats
							commit_dict['commit'] = commit.commit
							commit_dict['committer'] = commit.committer
							commit_dict['comments_url'] = commit.comments_url
							commit_dict['html_url'] = commit.html_url
							commit_dict['parents'] = commit.parents
							commit_dict['url'] = commit.url
							commit_list.append(commit_dict)

							num_of_commits += 1
							print(num_of_commits)

						with open(args.org + "/" + repo_name+"/"+args.event_type+"/"+branch+"_branch/" +  args.org + "-" +
					 		repo_name + "-"+branch+"_branch-" + args.event_type + "-page-" + str(page) + ".json", 'w') as f:
							f.write(str(commit_list))

						print("page ", page, " added to file")
						self.sleeper()
						page += 1

			print("commit data successfully collected")
		except Exception as e:
			print("Problem Occured: ", e)

	def collect_commitsComments(self, args):
		# call a get_repo function
		repo_list = self.get_repo(args)
		print("\n\tRepositories\n", repo_list)
		try:
			for repo_name in repo_list:
				print("\n\t" + repo_name + " Repository")
				repo = self.organization.get_repo(repo_name)

				totalCommitComments = repo.get_comments().totalCount
				print("total number of commit comments in ", repo_name, " Repository is", totalCommitComments)

				totalPage = totalCommitComments / args.per_page
				if totalPage is not int:
					totalPage = math.ceil(totalPage)
				print("total number of page with per_page ", self.per_page, " is ", totalPage)

				page = 0
				num_of_comments = 0

				while page < totalPage:
					comment_list = []
					print("\n\tpage: ", page)
					for comment in repo.get_comments():
						comment_dict = {}
						comment_dict['id'] = comment.id
						comment_dict['user'] = comment.user
						comment_dict['commit_id'] = comment.commit_id
						comment_dict['body'] = comment.body
						comment_dict['url'] = comment.url
						comment_dict['position'] = comment.position
						comment_dict['path'] = comment.path
						comment_dict['line'] = comment.line
						comment_dict['created_at'] = comment.created_at
						comment_dict['updated_at'] = comment.updated_at
						comment_dict['html_url'] = comment.html_url

						comment_list.append(comment_dict)

						num_of_comments += 1
						print(num_of_comments)

					with open(args.org + "/" + repo_name + "/" + args.event_type + "/" + args.org + "-" + repo_name + "-" +
							   args.event_type + "-page-" + str(page) + ".json", 'w') as f:
						f.write(str(comment_list))

					print("page ", page, " added to file")
					self.sleeper()
					page += 1

			print("data successfully collected")
		except Exception as e:
			print("Problem Occured: ", e)

	def collect_events(self, args):

		# call a get_repo function
		repo_list = self.get_repo(args)
		print("\n\tRepositories\n", repo_list)
		try:
			for repo_name in repo_list:
				repo = self.organization.get_repo(repo_name)
				total_events = repo.get_events().totalCount
				print("total number of Events in ", repo_name , " Repository is ", total_events)

				total_page = total_events / args.per_page
				if total_page is not int:
					total_page = math.ceil(total_page)
				print("The total number of page is: " + str(total_page))

				page = 0
				num_of_events = 0
				while page < total_page:
					event_list = []
					print("\n\tpage: ", page)
					for event in repo.get_events().get_page(page):
						event_dict = {}
						event_dict['actor'] = event.actor
						event_dict['id'] = event.id
						event_dict['payload'] = event.id
						event_dict['created_at'] = event.created_at
						event_dict['org'] = event.org
						event_dict['public'] = event.public
						event_dict['repo'] = event.repo
						event_dict['type'] = event.type
						event_list.append(event_dict)

						num_of_events += 1
						print(num_of_events)

					with open(args.org + "/" + repo_name + "/" + args.event_type + "/" + args.org + "-" + repo_name + "-" +
							  args.event_type + "-page-" + str(page) +  ".json", 'w') as f:
						f.write(str(event_list))

					print("page ", page, " added to file")
					self.sleeper()
					page += 1

			print("data successfully collected")
		except Exception as e:
			print("Problem Occured: ", e)

	def collect_issues_comments(self, args):
		repo_list = self.get_repo(args)
		print("\nRepositories:\n ", repo_list)
		try:
			for repo_name in repo_list:
				print("\n\t" + repo_name + " Repository")
				repo = self.organization.get_repo(repo_name)

				totalIssueComments = repo.get_issues_comments().totalCount
				print("total number of issue comments in ", repo_name, " Repository is", totalIssueComments)

				totalPage = totalIssueComments / args.per_page
				if totalPage is not int:
					totalPage = math.ceil(totalPage)
				print("total number of page with per_page ", self.per_page, " is ", totalPage)

				page = 0
				num_of_issue_comments = 0

				while page < totalPage:
					issue_comment_list = []
					print("\n\tpage: ", page)
					for comment in repo.get_issues_comments().get_page(page):
						comment_dict = {}
						comment_dict['id'] = comment.id
						comment_dict['user'] = comment.user
						comment_dict['body'] = comment.body
						comment_dict['issue_url'] = comment.issue_url
						comment_dict['created_at'] = comment.created_at
						comment_dict['updated_at'] = comment.updated_at
						comment_dict['url'] = comment.url
						comment_dict['html_url'] = comment.html_url

						issue_comment_list.append(comment_dict)

						num_of_issue_comments += 1
						print(num_of_issue_comments)

					with open(args.org + "/" + repo_name + "/" + args.event_type + "/" + args.org + "-" + repo_name + "-" +
							  args.event_type + "-page-" + str(page) + ".json", 'w') as f:
						f.write(str(issue_comment_list))

					print("page ", page, " added to file")
					self.sleeper()
					page += 1

			print("data successfully collected")
		except Exception as e:
			print("Problem occured:", e)

	def collect_issues(self, args):
		"""collect the data of the issue event in a given repository may be all repository or one repository"""
		# call a get_repo function
		repo_list = self.get_repo(args)
		print("\n\tRepositories\n", repo_list)
		try:
			for repo_name in repo_list:
				print("\n\t" + repo_name + " Repository")
				repo = self.organization.get_repo(repo_name)

				totalIssues = repo.get_issues(state=args.state).totalCount
				print("total number of " + args.state + " issues in " + repo_name + " Repository is: " , totalIssues)

				totalPage = totalIssues / args.per_page
				if totalPage is not int:
					totalPage = math.ceil(totalPage)
				print("total number of page with per_page ", self.per_page, " is ", totalPage)

				page = 0
				num_of_issue = 0

				while page < totalPage:
					issue_comment_list = []
					print("\n\tpage: ", page)
					issue_list = []
					for issue in repo.get_issues(state=args.state).get_page(page):
						issue_dict = {}
						issue_dict['number'] = issue.number
						issue_dict['id'] = issue.id
						issue_dict['user'] = issue.user
						issue_dict['title'] = issue.title
						issue_dict['body'] = issue.body
						issue_dict['url'] = issue.url
						issue_dict['milestone'] = issue.milestone
						issue_dict['labels'] = issue.labels
						issue_dict['labels_url'] = issue.labels_url
						issue_dict['created_at'] = issue.created_at
						issue_dict['updated_at'] = issue.updated_at
						issue_dict['closed_at'] = issue.closed_at
						issue_dict['closed_by'] = issue.closed_by
						issue_dict['pull_request'] = issue.pull_request
						issue_dict['state'] = issue.state
						issue_dict['events_url'] = issue.events_url
						issue_dict['comments'] = issue.comments
						issue_dict['number_of_comments'] = issue.comments
						issue_dict['comments_url'] = issue.comments_url
						issue_dict['assignee'] = issue.assignee
						issue_dict['assignees'] = issue.assignees
						issue_dict['html_url'] = issue.html_url

						issue_list.append(issue_dict)

						num_of_issue += 1
						print(num_of_issue)

					with open(args.org + "/" + repo_name + "/" + args.event_type + "/" + args.org + "-" + repo_name + "-" +
							  args.state + "-" + args.event_type + "-page-" + str(page) + ".json", 'w') as f:
						f.write(str(issue_list))

					print("page ", page, " added to file")
					self.sleeper()
					page += 1

			print("data successfully collected")
		except Exception as e:
			print("Problem Occured: ", e)

	def sleeper(self):
        # add sleeping time after requesting each page
		sleep_time = self.sleep_time_length + self.sleep_time_range * random.random()
		print("\n\tSleeping for", sleep_time, "seconds.")
		time.sleep(sleep_time)

	def main(self):

		# get the arguments from the terminal
		args = self.get_arguments()

		# validate the given arguments
		self.validate_arguments(args)

		# then create a github instance
		self.create_github_instance(args)

		# then call a create directory function
		self.create_directory(args)

		# identify the event type needed by the user and then go forward
		self.identify_event(args)

# Initialize the class
collect = Collect()
collect.main()


# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

from django.conf import settings
from django.shortcuts import render

# Create your views here.
def home(request):

    title = 'Welcome!'
    if request.user.is_authenticated():
        title = 'Welcome %s' %(request.user)

    context = {
               "title":title,
               }

    return render(request, "/studentsdb/templates/groups.html",context)


def run_tornado(self, args):
        """ Tornado dev server implementation """
        server = self
        import tornado.ioloop
        import tornado.web
        import tornado.websocket

        ioloop = tornado.ioloop.IOLoop.current()

        class DevWebSocketHandler(tornado.websocket.WebSocketHandler):
            def open(self):
                super(DevWebSocketHandler, self).open()
                server.on_open(self)

            def on_message(self, message):
                server.on_message(self, message)

            def on_close(self):
                super(DevWebSocketHandler, self).on_close()
                server.on_close(self)

        class MainHandler(tornado.web.RequestHandler):
            def get(self):
                self.write(server.index_page)

        #: Set the call later method
        server.call_later = ioloop.call_later
        server.add_callback = ioloop.add_callback

        app = tornado.web.Application([
            (r"/", MainHandler),
            (r"/dev", DevWebSocketHandler),
        ])

        app.listen(self.port)
        print("Tornado Dev server started on {}".format(self.port))
        ioloop.start()