from pathlib import Path

def main():
    workingdirectory = Path.cwd()

    #here we define an empty array:
    newfilelist = []

    #languages is a python dictionary
    #left value is the abbreviation we have in our filesnames
    #right value is the function name in our code that handles that language
    languages = {
        'ToRu' : 'convertToRussian',
        'ToCs' : 'convertToCzech',
        'ToDe' : 'convertToGerman',
        'ToNl' : 'convertToDutch',
        'ToEs' : 'convertToSpanish'
    }

    #here we get a list of files from the inputfiles directory
    #and go through this list and examine each filename for signs that it needs special processing.
    for filename in workingdirectory.glob('InputFiles/*.txt'):

        #for each file we have, we don't yet know if we'll need all translations
        #so we will set this true for the file once here
        #and down below, if it becomes unnecessary, we will set it false below
        ThisFileNeedsAllTranslations = True

        #break the filename into it's pieces ie Catalog_ToRu becomes 2 pieces Catalog + ToRu
        parts = filename.stem.split("_")

        for part in parts:

            # here we compare the part we have
            # (for example 'Catalog' or 'Ru" to our languages dictionary we defined above)
            # to what is in the dictionary
            if part in languages:
                # - if it's found (ie ToRu)
                # we use the globals() functon of python to call the function listed in the dictionary
                # ex: if 'part' = 'ToRu' then [languages[part]] will be set to the string 'convertToRussia' and the globals function will call the convertToRussia
                globals()[languages[part]](filename)

                #since we called a specific language, we assume this file doesn't need all translations:
                ThisFileNeedsAllTranslations = False

        if ThisFileNeedsAllTranslations:
            convertToLanguages(filename, languages)

#end main function

#Conversion functions
#could these be reduced to a single function with the language passed in?
#perhapse the langage codes used by the translator could be the abbreviated codes used in the file names?
def convertToRussian(file):
    print("converting " + file.stem + " to Russian")
    #conversion logic here
    #create file.stem.mp3 in output directory

def convertToCzech(file):
    print("converting " + file.stem + " to Czech")
    #conversion logic here
    #create file.stem.mp3 in output directory

def convertToGerman(file):
    print("converting " + file.stem + " to German")
    #conversion logic here
    #create file.stem.mp3 in output directory

def convertToDutch(file):
    print("converting " + file.stem + " to Dutch")
    #conversion logic here
    #create file.stem.mp3 in output directory

def convertToSpanish(file):
    print("converting " + file.stem + " to Spanish")
    #conversion logic here
    #create file.stem.mp3 in output directory

def convertToLanguages(file,languages):
    print("well look here, we have a file that needs to be converted to ALL languages!")
    for languagefunction in languages.values():
        globals()[languagefunction](file)

main() #begin program by calling main function


import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, roc_auc_score, auc, confusion_matrix
import numpy as np
import re
import os

def print_confusion_as_csv(eng_abbrs, confusion_matrix):
    _confusion_matrix = confusion_matrix.tolist()

    LEN = len(eng_abbrs)
    print('-, ', end='')
    for abbr in eng_abbrs:
        print(abbr ,end=', ')
    print('')
    for i in range(LEN):
        print(eng_abbrs[i] ,end=', ')
        for j in range(LEN):
            print(_confusion_matrix[i][j] ,end=', ')
        print('')

def draw_auc(targets, predicts, savefig_filename='auc.png'):
    # print(savefig_filename)
    # print(targets)
    # print(predicts)
    fpr, tpr, threshold = roc_curve(targets, predicts)
    #print(fpr, tpr, threshold)

    auc1 = auc(fpr, tpr)
    ## Plot the result
    plt.title('Receiver Operating Characteristic')
    plt.plot(fpr, tpr, color = 'orange', label = 'AUC = %0.2f' % auc1)
    plt.legend(loc = 'lower right')
    plt.plot([0, 1], [0, 1],'r--')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    plt.savefig(savefig_filename)
    plt.close()

def total_hit_topk(outputs, label, k):
    hit = 0
    for row in range(len(outputs)):
        # top k
        _, indices = outputs[row].topk(k, dim = 0, largest = True, sorted = True)
        for j in range(len(indices)):
            if label[row][indices[j].item()].item() == 1.0:
                hit += 1
                break
    return hit

def cal_confusion_matrix(targets, predicts):
    target_class , predict_class = [], []
    for t,p in zip(targets, predicts):
        target_class.append(t.index(max(t)))
        predict_class.append(p.index(max(p)))

    cm = confusion_matrix(target_class, predict_class)

    return np.matrix(cm)

def output_predict_result_sgfs(stat, eng_abbrs, tags_ch):
    TagCPattern = '[^AP]C\[([^\]]*)\]'
    for i , (p,t,sgf) in enumerate(zip(stat['predicts'],stat['target'], stat['sgf'])):
        comment = re.findall(TagCPattern, sgf)[-1]
        sgfstr = sgf.replace(comment, 'predict:{}  target:{}'.format(tags_ch[ p.index(max(p)) ] ,tags_ch[t.index(max(t))]))

        with open(os.path.join('../tempsgf',str(i)+'.sgf'), 'w') as fout:
            fout.write(sgfstr)


import random
import numpy as np

from torch.utils.data.dataset import Dataset

from src.datasets.bengali import IMAGE_KEY, INPUT_KEYS
from src.utils.utils import rand_bbox, to_one_hot

class CutMixUpDataset(Dataset):
    def __init__(self,
                 dataset,
                 image_key,
                 input_keys,
                 num_classes,
                 num_mix=1,
                 beta=1.,
                 prob=1.0):

        self.dataset = dataset
        self.num_mix = num_mix
        self.beta = beta
        self.prob = prob

        self.image_key = image_key
        self.input_keys = input_keys
        self.num_classes = num_classes

    def __getitem__(self, index):
        sample = self.dataset[index]
        img = sample[self.image_key]

        for _ in range(self.num_mix):
            r = np.random.rand(1)
            if self.beta <= 0 or r > self.prob:
                continue

            # generate mixed sample
            lam = np.random.beta(self.beta, self.beta)
            rand_index = random.randint(a=0, b=len(self))

            sample_ = self.dataset[rand_index]
            img_ = sample_[self.image_key]

            height, width = img_.size()[-2:-1]

            l, t, r, b = rand_bbox(lam, height, width)

            sample[self.image_key][:, t:b, l:r] = \
                img_[self.image_key][:, t:b, l:r]

            lam = 1 - float((r - l) * (b - t)) / (height * width)

            for input_key, num_classes in zip(self.input_keys, self.num_classes):
                sample[input_key] = lam * to_one_hot(sample[input_key], num_classes) + \
                            (1. - lam) * to_one_hot(sample_[input_key], num_classes)
        return img

    def __len__(self):
        return len(self.dataset)


# @Time    : 2021/5/25 17:28
# @Author  : lucas
# @File    : auth_gitee_request.py
# @Project : SSO
# @Software: PyCharm
import requests
from request.auth_base_request import AuthBaseRequest
from auth_source import AuthSource
from utils.utils import build_url

class AuthGiteeRequest(AuthBaseRequest):
    # https://gitee.com/api/v5/oauth_doc#/
    source = AuthSource.GITEE

    def __init__(self, config):
        self.client_id = config.get("client_id")
        self.client_secret = config.get("client_secret")
        self.redirect_uri = config.get("redirect_uri")
        self.refresh_token = ""

    def get_access_token(self, code):
        body = {
            "grant_type": "authorization_code",
            "client_id": self.client_id,
            "redirect_uri": self.redirect_uri,
            "client_secret": self.client_secret,
            "code": code
        }
        response = requests.post(self.source.get("access_token"), json=body)
        result = response.json()
        self.refresh_token = result.get("refresh_token")
        return result.get("access_token")

    def get_user_info(self, access_token):
        params = {"access_token": access_token}
        response = requests.get(build_url(self.source.get("user_info"), params=params))
        result = response.json()
        return result

    def get_refresh_token(self):
        params = {"grant_type": "refresh_token",
                  "refresh_token": self.refresh_token}
        response = requests.get(build_url(self.source.get("access_token"), params=params))
        result = response.json()
        print(result)

    def authorize(self):
        params = {
            "client_id": self.client_id,
            "redirect_uri": self.redirect_uri,
            "response_type": "code"
        }
        return build_url(self.source.get("authorize"), params)

if __name__ == "__main__":
    from auth_config import AuthConfig
    gitee = AuthGiteeRequest(AuthConfig.GITEE)
    print(gitee.authorize())


__author__ = 'Vas Vasiliadis <vas@uchicago.edu>'

import uuid
import time
import json
from datetime import datetime
from dateutil import tz
from dateutil import parser

import boto3
from boto3.dynamodb.conditions import Key
from botocore.client import Config
from botocore.exceptions import ClientError
from botocore.exceptions import ParamValidationError

from flask import (abort, flash, redirect, render_template,
  request, session, url_for)

from gas import app, db
from decorators import authenticated, is_premium
from auth import get_profile, update_profile

"""Start annotation request
Create the required AWS S3 policy document and render a form for
uploading an annotation input file using the policy document.

Note: You are welcome to use this code instead of your own
but you can replace the code below with your own if you prefer.
"""
@app.route('/annotate', methods=['GET'])
@authenticated
def annotate():
  # Create a session client to the S3 service
  s3 = boto3.client('s3',
    region_name=app.config['AWS_REGION_NAME'],
    config=Config(signature_version='s3v4'))

  bucket_name = app.config['AWS_S3_INPUTS_BUCKET']
  user_id = session['primary_identity']

  # Generate unique ID to be used as S3 key (name)
  key_name = app.config['AWS_S3_KEY_PREFIX'] + user_id + '/' + \
    str(uuid.uuid4()) + '~${filename}'

  # Create the redirect URL
  redirect_url = str(request.url) + '/job'

  # Define policy fields/conditions
  encryption = app.config['AWS_S3_ENCRYPTION']
  acl = app.config['AWS_S3_ACL']
  fields = {
    "success_action_redirect": redirect_url,
    "x-amz-server-side-encryption": encryption,
    "acl": acl
  }
  conditions = [
    ["starts-with", "$success_action_redirect", redirect_url],
    {"x-amz-server-side-encryption": encryption},
    {"acl": acl}
  ]

  # Generate the presigned POST call
  try:
    presigned_post = s3.generate_presigned_post(
      Bucket=bucket_name,
      Key=key_name,
      Fields=fields,
      Conditions=conditions,
      ExpiresIn=app.config['AWS_SIGNED_REQUEST_EXPIRATION'])
  except ClientError as e:
    app.logger.error(f"Unable to generate presigned URL for upload: {e}")
    return abort(500)

  # Render the upload form which will parse/submit the presigned POST
  return render_template('annotate.html', s3_post=presigned_post)

"""Fires off an annotation job
Accepts the S3 redirect GET request, parses it to extract
required info, saves a job item to the database, and then
publishes a notification for the annotator service.

Note: Update/replace the code below with your own from previous
homework assignments
"""
@app.route('/annotate/job', methods=['GET'])
@authenticated
def create_annotation_job_request():

  #Connect to DynamoDB table
  dynamodb = boto3.resource('dynamodb', region_name=app.config['AWS_REGION_NAME'])
  ann_table = dynamodb.Table(app.config['AWS_DYNAMODB_ANNOTATIONS_TABLE'])

  # Get bucket name, key, and job ID from the S3 redirect URL
  bucket_name = str(request.args.get('bucket'))
  s3_key = str(request.args.get('key'))
  user_id = session['primary_identity']

  #Parsing for the bucket, key, and jobID
  rule = request.url
  rule = str(rule)

  try:
      bucket_name_temp = rule.split("bucket=")[1]
      bucket_name = bucket_name_temp.partition('&')[0]
      key_temp = bucket_name_temp.split('&key=')[1]
      key = key_temp.partition("%")[0]
      job_id_temp = key_temp.partition("%")[2]
      job_id_temp = job_id_temp.partition("%")[2]
      job_id_temp = job_id_temp.partition("2F")[2]
      job_id_temp = job_id_temp.partition("&")[0]
      job_id = job_id_temp.partition('~')[0]
      input_file = job_id_temp.partition('~')[2]
      key_input_file = key + "/"+user_id+"/"+job_id+"~"+input_file

  except:
      print("The specified bucket or key variables could not be parsed in the url.")
      raise

  #Getting current time in epoch
  t = time.localtime()
  now = datetime.now()
  dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
  pattern = '%d/%m/%Y %H:%M:%S'
  #https://stackoverflow.com/questions/7241170/how-to-convert-current-date-to-epoch-timestamp
  epoch = int(time.mktime(time.strptime(dt_string, pattern)))

  email = session.get('email')

  #Data to be persisted in DynamoDB
  data = { 'job_id': job_id, 'user_id': user_id, 'input_file_name': input_file,
  "s3_inputs_bucket": bucket_name,
  "s3_key_input_file": key_input_file,
  "submit_time": epoch,
  "email": email,
  "job_status": "PENDING",
  }

  #Try to put data into the Database
  #Source - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.put_item
  try:
      ann_table.put_item(Item = data)
  except ClientError as e:
      print(e.response)
      raise

  #Source - https://www.edureka.co/community/55307/python-code-to-publish-a-message-to-aws-sns

  #Publishing a notification with data being persisted in DynamoDB to SNS topic to request annotator job
  sns = boto3.client('sns', region_name = app.config['AWS_REGION_NAME'])
  try:
      response  = sns.publish(

          TopicArn = app.config['AWS_SNS_JOB_REQUEST_TOPIC'],
          #Source - https://stackoverflow.com/questions/34029251/aws-publish-sns-message-for-lambda-function-via-boto3-python2
          Message = json.dumps({'default': json.dumps(data)}),
          MessageStructure='json',
      )
  except (ClientError, ParamValidationError) as e:
      print(e.response)
      raise

  #If the user is a free user, then send a notification to archive the file
  if session['role'] == "free_user":
    try:
      response  = sns.publish(

          TopicArn = app.config['AWS_SNS_ARCHIVE_TOPIC'],
          #Source - https://stackoverflow.com/questions/34029251/aws-publish-sns-message-for-lambda-function-via-boto3-python2
          Message = json.dumps({'default': json.dumps(data)}),
          MessageStructure='json',
      )
    except (ClientError, ParamValidationError) as e:
        print(e.response)
        raise

  return render_template('annotate_confirm.html', job_id=job_id)

"""List all annotations for the user
"""
@app.route('/annotations', methods=['GET'])
@authenticated
def annotations_list():

  #Connect to DynamoDB annotations table
  dynamodb = boto3.resource('dynamodb', region_name= app.config['AWS_REGION_NAME'])

  ann_table = dynamodb.Table(app.config['AWS_DYNAMODB_ANNOTATIONS_TABLE'])
  user_id = session['primary_identity']
  try:
    #Source - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.Python.04.html
    response = ann_table.query(
      IndexName = "user_id_index",
      KeyConditionExpression=Key('user_id').eq(user_id),
    )
  except ClientError as e:
    print(e.response)
    raise

  response_list = []
  to_zone = tz.tzlocal()

  for i in response['Items']:
      temp_dict = {}
      temp_dict['job_id'] = i['job_id']
      epoch_time = i['submit_time']
      #Convert submit time to given datetime format
      #Source - https://stackoverflow.com/questions/12400256/converting-epoch-time-into-the-datetime
      temp_dict['submit_time']= time.strftime('%Y-%m-%d %H:%M', time.localtime(epoch_time))
      temp_dict['input_file_name'] = i['input_file_name']
      temp_dict['job_status'] = i['job_status']
      response_list.append(temp_dict)

  # Get list of annotations to display

  return render_template('annotations.html', annotations=response_list)

"""Display details of a specific annotation job
"""
@app.route('/annotations/<id>', methods=['GET'])
@authenticated
def annotation_details(id):

  sns = boto3.client('sns', region_name = app.config['AWS_REGION_NAME'])

  dynamodb = boto3.resource('dynamodb', region_name= app.config['AWS_REGION_NAME'])
  ann_table = dynamodb.Table(app.config['AWS_DYNAMODB_ANNOTATIONS_TABLE'])
  user_id = session['primary_identity']

  try:
    #Source - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.query
    response = ann_table.query(
      KeyConditionExpression=Key('job_id').eq(id)
    )
  except ClientError as e:
    print(e.response)
    raise

  response_list = []
  for i in response['Items']:
    #If the given job is not currently running, then check for output files
    if i['job_status'] != "RUNNING":
      s3_key_results_file = i['s3_key_results_file']

      s3 = boto3.client('s3',
        region_name=app.config['AWS_REGION_NAME'],
        config=Config(signature_version='s3v4'))

      bucket_name = app.config['AWS_S3_RESULTS_BUCKET']
      key_name = s3_key_results_file

      #If premium user, and restore for file in progress, then show Message
      s3 = boto3.resource('s3')
      #Source - https://stackoverflow.com/questions/15085864/how-to-upload-a-file-to-directory-in-s3-bucket-using-boto
      try:
          s3.Object(bucket_name, key_name).load()
      except ClientError as e:
          if e.response['Error']['Code'] == "404":
              # The object does not exist.
              restore_message = True
          else:
              # Something else has gone wrong.
              raise
      else:
          # The object does exist.
          restore_message = False

      #If the object does exist then get presigned url for download
      #Source - https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-presigned-urls.html
      if restore_message == False:
        try:
          s3 = boto3.client('s3',
          region_name=app.config['AWS_REGION_NAME'],
          config=Config(signature_version='s3v4'))
          presigned_url = s3.generate_presigned_url(
            'get_object',
            Params = { 'Bucket': bucket_name,
            'Key': key_name},
            ExpiresIn=app.config['AWS_SIGNED_REQUEST_EXPIRATION'])
        except ClientError as e:
          app.logger.error(f"Unable to generate presigned URL for upload: {e}")
          return abort(500)
      else:
        presigned_url = ""

      #Parse the response for other parameters such as submit_time, complete_time etc.
      temp_dict = {}
      temp_dict['job_id'] = i['job_id']
      epoch_submit_time = i['submit_time']
      temp_dict['submit_time']= time.strftime('%Y-%m-%d %H:%M', time.localtime(epoch_submit_time))
      temp_dict['input_file_name'] = i['input_file_name']
      temp_dict['job_status'] = i['job_status']
      epoch_complete_time = i['complete_time']
      temp_dict['complete_time']= time.strftime('%Y-%m-%d %H:%M', time.localtime(epoch_complete_time))
      temp_dict['s3_key_results_file'] = i['s3_key_results_file']
      datetime_object = parser.parse(temp_dict['complete_time'])
      #If the user is a free user check if 5 minutes has passed since complete time to determine access expiry
      if (session.get('role') == "free_user"):
        a = datetime.now()
        difference = a - datetime_object
        seconds_in_day = 24 * 60 * 60
        #source - https://stackoverflow.com/questions/1345827/how-do-i-find-the-time-difference-between-two-datetime-objects-in-python
        x = divmod(difference.days * seconds_in_day + difference.seconds, 60)
        if x[0]>5:
          free_access_expired = True
        else:
          free_access_expired = False
      else:
        free_access_expired = False

      #If there is a restore message then display it
      if restore_message == True:
        temp_dict['restore_message'] = 'The file is currently restoring!'
      else:
        temp_dict['result_file_url'] = presigned_url

      response_list.append(temp_dict)
    else:
      #If the job is still running get the relevant parameters.
      free_access_expired = False
      temp_dict = {}
      temp_dict['job_id'] = i['job_id']
      epoch_submit_time = i['submit_time']
      temp_dict['submit_time']= time.strftime('%Y-%m-%d %H:%M', time.localtime(epoch_submit_time))
      temp_dict['input_file_name'] = i['input_file_name']
      temp_dict['job_status'] = i['job_status']
      response_list.append(temp_dict)

  return render_template('annotation_details.html', annotation=response_list[0], free_access_expired = free_access_expired)

"""Display the log file contents for an annotation job
"""
@app.route('/annotations/<id>/log', methods=['GET'])
@authenticated
def annotation_log(id):

  dynamodb = boto3.resource('dynamodb', region_name= app.config['AWS_REGION_NAME'])
  ann_table = dynamodb.Table(app.config['AWS_DYNAMODB_ANNOTATIONS_TABLE'])

  try:
    #Source - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.query
    response = ann_table.query(
      KeyConditionExpression=Key('job_id').eq(id)
    )
  except ClientError as e:
    print(e.response)
    raise
  #Query using the job id and get the log file name
  for i in response['Items']:
    s3_key_log_file = i['s3_key_log_file']

  bucket= app.config['AWS_S3_RESULTS_BUCKET']
  key = s3_key_log_file

  #Source - https://stackoverflow.com/questions/31976273/open-s3-object-as-a-string-with-boto3
  s3 = boto3.resource('s3')
  obj = s3.Object(bucket, key)
  #Read the log file
  response = obj.get()['Body'].read().decode('utf-8')

  return render_template('view_log.html', job_id = id, log_file_contents=response)

"""Subscription management handler
"""
import stripe

@app.route('/subscribe', methods=['GET', 'POST'])
@authenticated
def subscribe():
  if (request.method == 'GET'):
    # Display form to get subscriber credit card info
    if (session.get('role') == "free_user"):
      return render_template('subscribe.html')
    else:
      return redirect(url_for('profile'))

  elif (request.method == 'POST'):
    # Process the subscription request
    token = str(request.form['stripe_token']).strip()

    # Create a customer on Stripe
    stripe.api_key = app.config['STRIPE_SECRET_KEY']
    try:
      customer = stripe.Customer.create(
        card = token,
        plan = "premium_plan",
        email = session.get('email'),
        description = session.get('name')
      )
    except Exception as e:
      app.logger.error(f"Failed to create customer billing record: {e}")
      return abort(500)

    # Update user role to allow access to paid features
    update_profile(
      identity_id=session['primary_identity'],
      role="premium_user"
    )

    # Update role in the session
    session['role'] = "premium_user"

    # Request restoration of the user's data from Glacier
    # Add code here to initiate restoration of archived user data
    # Make sure you handle files not yet archived!

    user_id = session['primary_identity']

    dynamodb = boto3.resource('dynamodb', region_name= app.config['AWS_REGION_NAME'])
    ann_table = dynamodb.Table(app.config['AWS_DYNAMODB_ANNOTATIONS_TABLE'])
    #Query the table using the user_id to get all files associated with the user
    #Source - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.query
    try:
      response = ann_table.query(
        IndexName = "user_id_index",
        KeyConditionExpression=Key('user_id').eq(user_id),
      )
    except ClientError as e:
      print(e.response)
      raise

    for i in response['Items']:
      if 'results_file_archive_id' in i:
        #If there is any files for the user to be restored, restore them
        if i['results_file_archive_id'] != None and i['results_file_archive_id'] != "restored":
          results_file_archive_id = i['results_file_archive_id']
          s3_key_results_file = i['s3_key_results_file']
          s3_results_bucket = i['s3_results_bucket']

          data = {}
          data['results_file_archive_id'] = results_file_archive_id
          data['s3_key_results_file'] = s3_key_results_file
          data['s3_results_bucket'] = s3_results_bucket

          #Publish to restore SNS topic
          sns = boto3.client('sns', region_name = app.config['AWS_REGION_NAME'])
          try:
            response  = sns.publish(

                TopicArn = app.config['AWS_SNS_RESTORE_TOPIC'],
                #Source - https://stackoverflow.com/questions/34029251/aws-publish-sns-message-for-lambda-function-via-boto3-python2
                Message = json.dumps({'default': json.dumps(data)}),
                MessageStructure='json',
            )
          except (ClientError, ParamValidationError) as e:
              print(e.response)
              raise

    # Display confirmation page
    return render_template('subscribe_confirm.html',
      stripe_customer_id=str(customer['id']))

"""Reset subscription
"""
@app.route('/unsubscribe', methods=['GET'])
@authenticated
def unsubscribe():
  # Hacky way to reset the user's role to a free user; simplifies testing
  update_profile(
    identity_id=session['primary_identity'],
    role="free_user"
  )
  return redirect(url_for('profile'))

"""DO NOT CHANGE CODE BELOW THIS LINE
*******************************************************************************
"""

"""Home page
"""
@app.route('/', methods=['GET'])
def home():
  return render_template('home.html')

"""Login page; send user to Globus Auth
"""
@app.route('/login', methods=['GET'])
def login():
  app.logger.info(f"Login attempted from IP {request.remote_addr}")
  # If user requested a specific page, save it session for redirect after auth
  if (request.args.get('next')):
    session['next'] = request.args.get('next')
  return redirect(url_for('authcallback'))

"""404 error handler
"""
@app.errorhandler(404)
def page_not_found(e):
  return render_template('error.html',
    title='Page not found', alert_level='warning',
    message="The page you tried to reach does not exist. \
      Please check the URL and try again."
    ), 404

"""403 error handler
"""
@app.errorhandler(403)
def forbidden(e):
  return render_template('error.html',
    title='Not authorized', alert_level='danger',
    message="You are not authorized to access this page. \
      If you think you deserve to be granted access, please contact the \
      supreme leader of the mutating genome revolutionary party."
    ), 403

"""405 error handler
"""
@app.errorhandler(405)
def not_allowed(e):
  return render_template('error.html',
    title='Not allowed', alert_level='warning',
    message="You attempted an operation that's not allowed; \
      get your act together, hacker!"
    ), 405

"""500 error handler
"""
@app.errorhandler(500)
def internal_error(error):
  return render_template('error.html',
    title='Server error', alert_level='danger',
    message="The server encountered an error and could \
      not process your request."
    ), 500

### EOF


# - - coding:utf-8 - -
"""
# author: wktxzr
# Created Time : 20180503
# File Name: calculate_q_main.py
# Description:
计算IDF
"""

import os
import multiprocessing

CUR_PATH = os.path.split(os.path.realpath(__file__.split('/')[-1]))[0]
TOPICS = 'MB226 MB230 MB254 MB265 MB267 MB320 MB358 MB362 MB371 MB377 MB391 MB392 MB409 MB425'

def sys_order(topic):
	# 预处理后的推特数据
	data_path = CUR_PATH + '/init_data/'
	# 扩展词
	# dic[topic] = word_list
	topic_day_path = CUR_PATH + '/id_newsexword_dic_20.pkl'

	output_path = CUR_PATH + '/' + topic + '.pkl'

	order = 'python calculate_q.py' + ' -d ' + data_path + ' -o ' + \
	output_path + ' -i ' + topic_day_path +  ' -w ' + topic + ' -k ' + \
	k1 +  ' -b ' + b +  ' -v ' + avgdl
	os.system(order)

if __name__ == '__main__':
	pool = multiprocessing.Pool(processes = 7)
	topic_list = TOPICS.split(' ')
	for topic in topic_list:
		pool.apply_async(sys_order, args = (topic, ))
	pool.close()
	pool.join()
	# topic_list = TOPICS.split(' ')
	# for topic in topic_list:
	# 	sys_order(topic)


from MainControlLoop.lib.drivers.APRS import APRS
from MainControlLoop.lib.StateFieldRegistry import StateFieldRegistry, StateField, StateFieldRegistryLocker
from MainControlLoop.lib.modes import Mode
from MainControlLoop.tasks.APRS.actuate_task import APRSActuateTask, APRSCriticalMessage
from MainControlLoop.tasks.DownLinkProducer import DownLinkProducer

import re
from enum import Enum

class BeaconInterval(Enum):
    FAST = 30
    SLOW = 120
    CUSTOM = -1
    NEVER = 1

class APRSCommands(Enum):
    BEACON = r"TJ:C;APRS;beacon;;"
    BEACON_INTERVAL = r"TJ:C;APRS;beacon_interval;\d+;"
    SFR = r"TJ:C;APRS;SFR;;"
    SFR_TIME = r"TJ:C;APRS;SFR_time;\d+;"
    SF = r"TJ:C;APRS;SF;[\w\d_]+;"
    RESET = r"TJ:C;APRS;reset;;"

    ANTENNA_DEPLOY_INITIATED = r"pFS:AntennaDeployer;DEPLOYED;"

class APRSControlTask:

    def __init__(self, aprs: APRS, state_field_registry: StateFieldRegistry, locker: StateFieldRegistryLocker,
                 actuate_task: APRSActuateTask):
        self.aprs: APRS = aprs
        self.state_field_registry: StateFieldRegistry = state_field_registry
        self.locker = locker
        self.mode: Mode = Mode.BOOT

        self.actuate_task: APRSActuateTask = actuate_task

    def execute(self, commands):
        # TODO: control task logic HAS NOT been decided for boot/startup/safe mode

        current_sys_time: float = self.state_field_registry.get(StateField.TIME)
        command_joined = ''.join(commands)  # join all commands from list to single string

        if self.mode == Mode.SAFE:  # switch off beacon if in safe mode
            self.state_field_registry.update(StateField.APRS_BEACON_INTERVAL, BeaconInterval.NEVER.value)
            return

        if self.mode == Mode.STARTUP: # if in startup mode and antenna has been deployed, enable a critical message that the antenna has been deployed
            if re.search(APRSCommands.ANTENNA_DEPLOY_INITIATED.value, command_joined) is not None:
                self.actuate_task.set_critical_message(APRSCriticalMessage.ANTS_DEPLOYED)
                self.actuate_task.enable_critical_message()

        if self.mode == Mode.LOW_POWER:  # set beacon interval to slow if in low power mode
            self.state_field_registry.update(StateField.APRS_BEACON_INTERVAL, BeaconInterval.SLOW.value)

        if self.mode == Mode.COMMS:  # if in comms mode, turn off beacon, enable aprs to dump statefield
            self.state_field_registry.update(StateField.APRS_BEACON_INTERVAL, BeaconInterval.NEVER.value)

            dump = self.locker.find(current_sys_time)
            self.actuate_task.set_dump(dump)
            self.actuate_task.enable_dump()

        if self.mode == Mode.NORMAL:  # if in normal mode, set beacon interval to fast
            self.state_field_registry.update(StateField.APRS_BEACON_INTERVAL, BeaconInterval.FAST.value)

        if re.search(APRSCommands.SFR.value, command_joined) is not None:  # if sfr method is in command_joined, create  adump for the current statefield
            dump = DownLinkProducer.create_dump(self.state_field_registry)
            self.actuate_task.set_dump(dump)
            self.actuate_task.enable_dump()

        match = re.search(APRSCommands.SFR_TIME.value, command_joined)
        if match is not None: # if sfr time in command_joined, find previous dump and enable dunp
            cmd = command_joined[match.start(): match.end()]
            arg = cmd.split(";")[-2]
            timestamp = float('0' + re.sub(r"[^\d.]", '', arg))

            dump = self.locker.find(timestamp)
            self.actuate_task.set_dump(dump)
            self.actuate_task.enable_dump()

        match = re.search(APRSCommands.SF.value, command_joined)
        if match is not None:
            cmd = command_joined[match.start(): match.end()]
            arg = cmd.split(";")[-2]
            try:
                state_field = StateField(arg)
                response = DownLinkProducer.create_response(self.state_field_registry, state_field)
                if response != '':
                    self.actuate_task.set_response(response)
                    self.actuate_task.enable_response()
            except:
                pass

        if re.search(APRSCommands.RESET.value, command_joined) is not None:
            self.state_field_registry.update(StateField.APRS_BEACON_INTERVAL, BeaconInterval.NEVER)
            self.actuate_task.enable_reset()

        interval = self.state_field_registry.get(StateField.APRS_BEACON_INTERVAL)
        last_beacon_time: float = self.state_field_registry.get(StateField.APRS_LAST_BEACON_TIME)

        if current_sys_time - last_beacon_time > interval or re.search(APRSCommands.BEACON.value,
                                                                       command_joined) is not None:  # create a beacon with state field registry if over time interval or beacon value in command
            beacon = DownLinkProducer.create_beacon(self.state_field_registry)
            self.actuate_task.set_beacon(beacon)
            self.actuate_task.enable_beacon()


# -*- coding: utf-8 -*-

import rdflib
import subprocess
import sys
import codecs

sys.stdout = codecs.getwriter("utf-8")(sys.stdout)

# On créé un graph RDF et on charge le contenu de la base de données Zotero
# dans ce graph
g = rdflib.Graph()
g.load('Zotero-AK.rdf')

# On déclare les espace de noms utilisés dans Zotero
ns = {
    'bib': rdflib.Namespace("http://purl.org/net/biblio#"),
    'z': rdflib.Namespace("http://www.zotero.org/namespaces/export#"),
    'dc': rdflib.namespace.DC,
    'foaf': rdflib.namespace.FOAF
}

qs = """SELECT ?doc ?title
WHERE {
    ?doc a bib:Document .
    ?doc z:itemType "blogPost" .
    ?doc dc:title ?title .
}
"""

results = g.query(qs, initNs=ns)

for res in results:
    sys.stdout.write(u"%s\t%s\n" % (res.doc[7:], res.title))
    #cmd = u"webtocon %s screenshots/%s.png 1280 1024" % (res.doc, res.title)
    #p1 = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    #(stdout_data, stderr_data) = p1.communicate()


"""
Functions and classes for sampling the *showq* output.
"""

import remote
from script     import Data_jobscript
from cfg        import Cfg
from qstatx     import Data_qstat
from sar        import Data_sar
from titleline  import title_line
import          rules
from mycollections import OrderedDict,od_add_list_item,od_last
from cluster    import current_cluster,cluster_properties

import pickle,os,shutil,gzip
from time       import sleep
import datetime

# list of users we want to ignore for the time being...
ignore_users = []

#===============================================================================
def run_showq():
    """
    1. Run command ``showq -r -p hopper --xml`` on a login node,
    2. Parse its xml output into an OrderedDict (xmltodict).

    Typical output of ``print(run_showq())`` ::

        OrderedDict([('Data', OrderedDict(
            [ ('Object', 'queue')
            , ('cluster', OrderedDict(
                            [ ('@LocalActiveNodes', '167')
                            , ('@LocalAllocProcs', '3305')
                            , ('@LocalConfigNodes', '168')
                            , ('@LocalIdleNodes', '1')
                            , ('@LocalIdleProcs', '54')
                            , ('@LocalUpNodes', '168')
                            , ('@LocalUpProcs', '3360')
                            , ('@RemoteActiveNodes', '0')
                            , ('@RemoteAllocProcs', '0')
                            , ('@RemoteConfigNodes', '0')
                            , ('@RemoteIdleNodes', '0')
                            , ('@RemoteIdleProcs', '0')
                            , ('@RemoteUpNodes', '0')
                            , ('@RemoteUpProcs', '0')
                            , ('@time', '1487065534')
                            ]))
            , ('queue', OrderedDict([ ('@count', '40')
                                    , ('@option', 'active')
                                    , ('job', [ <job entry OrderedDict for each job> ])
                                    ]))
            ]))
        ])

    """
    data_showq = remote.run("showq -r -p hopper --xml",post_processor=remote.xml_to_odict)
    return data_showq

#===============================================================================
class ShowqJobEntry:
    """
    Class for storing and manipulating a single job entry in the xml output of showq.

    Here is a typical job entry (in xml). It is converted to an :class:`OrderdDict`
    by :func:`xmltodict.parse`:

    .. code-block:: html

       <job
       AWDuration="192260"
       Class="q7d"
       DRMJID="393684.hopper"
       EEDuration="106250"
       GJID="393684"
       Group="vsc20213"
       JobID="393684"
       JobName="H2Diss-slabal2o3_MoO3"
       MasterHost="r3c4cn02.hopper.antwerpen.vsc"
       PAL="hopper"
       ReqAWDuration="604800"
       ReqProcs="160"
       RsvStartTime="1485577392"
       RunPriority="4970"
       StartPriority="4970"
       StartTime="1485577392"
       StatPSDed="30761912.000000"
       StatPSUtl="3691030.208000"
       State="Running"
       SubmissionTime="1485471138"
       SuspendDuration="0"
       User="vsc20213"
       >
       </job>
    """
    #---------------------------------------------------------------------------
    def __init__(self,job_entry):
        self.data = job_entry # OrderedDict
    #---------------------------------------------------------------------------
    def get_jobid_long(self):
        """
        :return: long jobid, includes the cluster on which it was submitted.
        :rtype: str
        """
        jobid = self.data['@DRMJID']
        return jobid
    #---------------------------------------------------------------------------
    def get_jobid(self):
        """
        :return: short jobid, just the number.
        :rtype: str
        """
        jobid = self.data['@JobID']
        return jobid
    #---------------------------------------------------------------------------
    def get_state(self):
        """
        :return: state of the job, 'R', 'C', ...
        :rtype: str, 1 character.
        """
        state = self.data['@State']
        return state
    #---------------------------------------------------------------------------
    def get_effic(self,ncores_used_on_mhost=0):
        """
        Return the efficiciency, corrected or uncorrected (*ncores_used_on_mhost*>0).

        :param int ncores_used_on_mhost: if larger than zero, represents the number of cores used on mhost and the corrected efficiency is computed (see below), otherwise the uncorrected efficiency is returned.

        According to the Adaptive Computing developers:

        "For active jobs Moab reads in the "resources_used.cput" from Torque and
        divides that by the total processor seconds dedicated to the job. This
        is the efficiency of the job. Note that this can be a delayed statistic
        coming from torque so EFFIC is just an estimate until after a job has
        finished."

        The efficiency is termed uncorrected because in the current setup of hopper
        Torque has no information from the slave nodes, and hence assumes that
        *resources_used.cput* is 0 for slave nodes. E.g. for a job running on 2 nodes,
        each with 100% efficiency, Moab will report an efficiency of 50% after 100s
        because it is computed as:

        - total processor seconds: 100s * 2 nodes * 20 cores per node = 4000
        - *resources_used.cput* on the master node = 100s * 20 cores per node = 2000
        - *resources_used.cput* on the slave  node = 100s * 20 cores per node = 2000,
          but Torque sees 0s * 20 cores per node = 0s and thus reports (2000+0)/4000 = 50%

        We can scale the efficiency to the master node as

        - 50% * number_of_cores_used_by_all_nodes / number_of_cores_used_by_master_node
          = 50% * 40/20 = 100%

        Note that the JobEntry by itself does NOT know the number_of_cores_used_by_master_node
        and thus cannot correct the effic value, unless this value is provided as ncores_used_on_mhost.

        The 'corrected' value, obviously provides only information on the master host node
        rather than on the entire job! It is our hope that if the master node is performing
        well, the slave nodes do so too.
        """
        remote.err_print('method showq.ShowqJobEntry.get_effic(self, ncores_used_on_mhost) may '
                         'report incorrect values. Use JobSample.get_effic() instead.'
                        , print_time=False
                        )
        numerator   = self.data['@StatPSUtl']
        denominator = self.data['@StatPSDed']
        try:
            value = 100*float(numerator)/float(denominator) # [%]
        except ZeroDivisionError:
            value = 100 # seems safe
        if ncores_used_on_mhost>0 and Cfg.correct_effic:
            # scale the effic value to the master host node only, i.e. "correct" it.
            value *= self.get_ncores()/ncores_used_on_mhost
        return round(value,2)
    #---------------------------------------------------------------------------
    def get_username(self):
        """
        :return: username of the user that started this job.
        """
#         :return str: username.
        value = self.data['@User']
        return value
    #---------------------------------------------------------------------------
    def get_mhost(self,short=True):
        """
        :return: the mhost node (node on wcich the job started.
        """
#         :return str: name of the master compute node.
        value = self.data['@MasterHost']
        if short:
            value = value.split('.',1)[0]
        return value
    #---------------------------------------------------------------------------
    def get_ncores(self):
        """
        :return int: total number of cores for this job .
        """
        value = int( self.data['@ReqProcs'] )
        return value
    #---------------------------------------------------------------------------

#===============================================================================
def overview_by_user(arg):
    """
    Sort key for sorting warnings by username
    """
    return arg.split(' ',1)[1]
    #---------------------------------------------------------------------------

#===============================================================================
timestamp_format = '%Y.%m.%d.%Hh%M'
#===============================================================================
def get_timestamp_now():
    """
    :return: a timestamp based on the current time.
    :rtype: str
    """
    timestamp = datetime.datetime.now().strftime(timestamp_format)
    return timestamp
    #---------------------------------------------------------------------------

#===============================================================================
class JobSample:
    """
    :param ShowqJobEntry job_entry: the job entry.
    :param Job job: the parent :class: Job object.
    :param str timestamp: the timestamp of the sample.
    """
    #---------------------------------------------------------------------------
    def __init__(self,job_entry,job,timestamp):
        assert isinstance(job_entry, ShowqJobEntry)
        assert isinstance(job, Job)
        self.showq_job_entry = job_entry
        self.parent_job      = job
        self.timestamp       = timestamp
        self.data_qstat      = Data_qstat( job.jobid )
        self.mhost_job_info  = None# NeighbouringJobInfo(self)
        self.data_sar        = None
        self.details = ''
    #---------------------------------------------------------------------------
    def check_for_issues(self):
        """
        :return: True (False) if there are (aren't) issues (all rules satisfied) for this JobSample
        """
        self.mhost_job_info  = NeighbouringJobInfo(self)
        self.warnings = []
        self.overview = ''
        self.details  = ''
        if not self.data_qstat.is_interactive_job(): #interactive jobs are ignored
            for irule,rule in enumerate(rules.the_rules):
                msg = rule.check(self)
                if msg:
                    self.warnings.append(msg)
                    self.parent_job.warning_counts[irule] += 1

        if self.warnings:
            self.parent_job.nsamples_with_warnings += 1
            return True
        else:
            return False
    #---------------------------------------------------------------------------
    def compose_overview(self):
        """
        :return: the overview text of the JobSample, composed when asked the first time.
        :rtype: str
        """
        if self.warnings:
            if self.overview:
                return self.overview
            desc = '{} {} {} {}|{}'.format( self.showq_job_entry.get_jobid()
                                          , self.showq_job_entry.get_username()
                                          , self.showq_job_entry.get_mhost()
                                          , self.get_nnodes()
                                          , self.get_ncores()
                                          )
            self.overview = '\n'+( desc.ljust(32)+self.warnings[0] ).ljust(68)+str(self.parent_job.jobscript.loaded_modules(short=True))
            spaces = '\n'+(32*' ')
            for w in self.warnings[1:]:
                w0 = w.split('\n',1)[0]
                self.overview += spaces+w0
        else:
            self.overview = ''
        return self.overview
    #---------------------------------------------------------------------------
    def compose_details(self):
        """
        :return: the details text of the JobSample, composed when asked the first time.
        :rtype: str
        """
        if self.details or not self.warnings:
            return self.details

        self.details = str(self.overview) # make a copy
        self.details += '\n\n#samples with warnings : {} / {} = {}%'.format( self.parent_job.nsamples_with_warnings
                                                                           , self.parent_job.nsamples()
                                                                           , round(100*self.parent_job.nsamples_with_warnings/self.parent_job.nsamples(),2)
                                                                           )
        for irule,count in enumerate(self.parent_job.warning_counts):
            rule = rules.the_rules[irule]
            if count>0:
                self.details +='\n  {:25}: {:5}'.format(rule.warning,count)

        self.details += '\nwalltime used/remaining: {} / {}'.format( self.data_qstat.get_walltime_used()
                                                                   , self.data_qstat.get_walltime_remaining()
                                                                   )
        mem_available = cluster_properties[current_cluster]['mem_avail_gb'](self.get_nodes())
        self.details += '\nmem [GB] used/requested/available: {} / {} / {}'.format( round(self.data_qstat.get_mem_used()     ,3)
                                                                                , round(self.data_qstat.get_mem_requested(),3)
                                                                                , mem_available
                                                                                )
        hdr = 'nodes and cores used: '
        nohdr = len(hdr)*' '
        nodes = self.data_qstat.get_exec_host().split('+')
        self.details += '\n'+hdr+nodes[0]
        for node in nodes[1:]:
            self.details += '\n'+nohdr+node

        self.details += self.mhost_job_info.to_str()

        if self.data_qstat.node_sar:
            self.details += title_line('sar -P ALL 1 1',width=100,char='-')
            if len(self.data_qstat.node_sar)>1:
                avgs = [self.get_ncores()]
                avgs.extend(6*[0])
                for data_sar in self.data_qstat.node_sar.values():
                    if hasattr(data_sar,'columns'):
                        avgs[1] += data_sar.columns['%user'  ][0]
                        avgs[2] += data_sar.columns['%nice'  ][0]
                        avgs[3] += data_sar.columns['%system'][0]
                        avgs[4] += data_sar.columns['%iowait'][0]
                        avgs[5] += data_sar.columns['%steal' ][0]
                        avgs[6] += data_sar.columns['%idle'  ][0]
                nnodes = self.get_nnodes()
                for i in range(1,7):
                    avgs[i]/=nnodes
                self.details += 'AVERAGE  '+Data_sar.line_fmt.format(*avgs)+'\n'
            for node, data_sar in self.data_qstat.node_sar.items():
                for line in data_sar.data_cores:
                    self.details += node+' '+line +'\n'
        self.details += title_line('Script',width=100,char='-')
        for line in self.parent_job.jobscript.clean:
            self.details += line+'\n'
        self.details += title_line(width=100,char='-')

        return self.details
    #---------------------------------------------------------------------------
    def walltime(self,hours=False):
        """
        :param bool hours: select the return type and format: *True->int* = #hours, *False->str* = HH:MM:SS.
        :return: the current walltime as reported by qstat, either as the number of hours *(int)*, or as HH:MM:SS *(str)*.
        :rtype: int or str.
        """
        try:
            wt = self.data_qstat.data['resources_used']['walltime']
            if hours:
                wt = '{:.2f} hrs'.format( hhmmss2s(wt)/3600 )
        except KeyError:
            if hours:
                wt = '? hrs'
            else:
                wt = '??:??:??'
        return wt
    #---------------------------------------------------------------------------
    def nodedays(self):
        """
        :return: the number of node days comsumed so far, as a formatted str.
        """
        try:
            wt = self.data_qstat.data['resources_used']['walltime']
            nd = hhmmss2s(wt)*self.get_nnodes()/(3600*24)
            nd = '{:.3f} node days'.format(nd)
        except KeyError:
            nd = '?'
        return nd
    #---------------------------------------------------------------------------
    def get_effic(self,mhost_only=Cfg.correct_effic):
        """
        Compute the efficiency from the qstat output as::

            effic = 100 * cputime_used_by_all_cores / (ncores*walltime) # percentage

        This should be close to 100% for well-performing jobs.

        :param bool mhost_only: if True, assumes that the reported efficiency is based on information from the mhost node only, an scales the reported value accordingly.
        :return: efficiency as a percentage.

        .. note:: If mhost_only is *True*, and this is a multi-node job, the value reurned
                  provides **only** information on the mhost node. If it is above the threshold,
                  it is assumed/hoped that the slave nodes are well-performing too, but there is
                  no guarantee. To be certain, the sar output must be inspected (but it is only
                  generated if the mhost is below the threshold).
                  See allso :func:`ShowqJobEntry.get_effic()`.
        """
        if not hasattr(self,'effic'):
            # we must first compute it.
            try:
                walltime                   = hhmmss2s( self.data_qstat.data['resources_used']['walltime'] )
                cputime_used_by_all_cores  = hhmmss2s( self.data_qstat.data['resources_used']['cput'] )
                ncores   = self.get_ncores()
                self.effic = 100*cputime_used_by_all_cores/(ncores*walltime)
                if mhost_only:
                    ncores_on_mhost = self.get_ncores(cn='mhost')
                    self.effic *= (ncores/ncores_on_mhost)
            except Exception as e:
                remote.err_print('JobSample.get_effic():',type(e),e)
                self.effic = 0

        return self.effic
    #---------------------------------------------------------------------------
    def get_ncores(self,cn='all'):
        """
        :param str cn: compute node name. If equal to 'mhost' the master node is taken.
        :return: number of cores in use on compute node *cn*. If *cn* is *None* the total number of cores in use
        :rtype: int
        """
        return self.data_qstat.node_cores.ncores(cnode=cn)
    #---------------------------------------------------------------------------
    def get_nnodes(self):
        """
        :return: the number of nodes used as reported by qstat.
        """
        return self.data_qstat.get_nnodes()
    #---------------------------------------------------------------------------
    def get_nodes(self):
        """
        :return: a list of the (short) node names on which the job is running.
        """
        return self.data_qstat.node_cores.nodes()
    #---------------------------------------------------------------------------
    def get_jobid(self):
        """
        :return: the jobid
        """
        return self.data_qstat.jobid
    #---------------------------------------------------------------------------
    def get_mhost(self,short=False):
        """
        Return the name of the mhost node, e.g. 'r5c2cn01.hopper.antwerpen.vsc'.
        If *short==True*, it is shortened to the part in front of the first dot: i.e.
        'r5c2cn01'.
        """
        if short:
            mhost = self.data_qstat.node_cores.mhost
        else:
            mhost = self.data_qstat.get_master_node()
        return mhost
    #---------------------------------------------------------------------------
    def get_mem(self):
        """
        :return: the maximum of memory used and requested.
        """
        memreqd = self.data_qstat.get_mem_requested()
        memused = self.data_qstat.get_mem_used()
        mem = max(memreqd,memused)
        return mem
    #---------------------------------------------------------------------------
#     def get_effic(self):
#         """
#         from showq or data_qstqt
#         """
#         if
#         if Cfg.correct_effic:
#             ncores_used_on_mhost = self.get_ncores(cn='mhost')
#         else:
#             ncores_used_on_mhost = 0
#         value = self.showq_job_entry.get_effic(ncores_used_on_mhost)
#         return value
    #---------------------------------------------------------------------------

#===============================================================================
class NeighbouringJobInfo:
    """
    Info on all jobs running on the master node of a job sample.

    :param JobSample job_sample:
    """
    def __init__(self,job_sample):
        timestamp = job_sample.timestamp
        jobid1 = job_sample.get_jobid()
        self.jobid  = [jobid1]
        self.nnodes = [job_sample.get_nnodes()]
        self.ncores = [job_sample.get_ncores()]
        self.effic  = [job_sample.get_effic ()]
        self.memory = [job_sample.get_mem   ()]

        self.mhost = job_sample.get_mhost(short=True)
        neighbouring_jobs = job_sample.parent_job.sampler.mhost_jobs[self.mhost]

        for jobid2 in neighbouring_jobs:
            if jobid2 != jobid1:
                job2 = job_sample.parent_job.sampler.jobs[jobid2]
                try:
                    job2sample = job2.get_sample(timestamp)
                except KeyError as e:
                    print(type(e),e,job2)
                    self.nnodes.append(0)
                    self.ncores.append(0)
                    self.effic .append(0)
                    self.memory.append(0)
                else:
                    self.nnodes.append(job2sample.get_nnodes())
                    self.ncores.append(job2sample.get_ncores())
                    self.effic .append(job2sample.get_effic ())
                    self.memory.append(job2sample.get_mem   ())
                self.jobid .append(jobid2)
        self.n = len(neighbouring_jobs)
        if self.n>1:
            self.jobid .append('total:')
            self.nnodes.append(1)
            self.ncores.append(sum(self.ncores))
            effic = 0
            for i in range(self.n):
                effic += self.effic[i]*self.ncores[i]
            self.effic.append( effic/self.ncores[-1] )
            self.memory.append(sum(self.memory))
    #---------------------------------------------------------------------------
    def to_str(self):
        """
        Format self as a *str*.
        """
        s = '\nother jobs on {}: '.format(self.mhost)
        if self.n == 1:
            s += 'None.'
        else:
            fmt = '\n  **{}**{:3}|{:2} {:5.1f}% {:7.3f}GB'
            i = 0
            s+= '{} (total={}).'.format(self.n-1,self.n)
            s+= fmt.format( self.jobid [i]
                          , self.nnodes[i]
                          , self.ncores[i]
                          , self.effic [i]
                          , self.memory[i]
                          )
            fmt = fmt.replace('*',' ')
            for i in range(1,self.n):
                if self.nnodes[i]!=0:
                    s += fmt.format( self.jobid [i]
                                   , self.nnodes[i]
                                   , self.ncores[i]
                                   , self.effic [i]
                                   , self.memory[i]
                                   )
                else:
                    s+= '\n    {} (no info)'.format(self.jobid[i])
        s+='\n'
        return s
    #---------------------------------------------------------------------------

#===============================================================================
def hhmmss2s(hhmmss):
    """
    Convert time duration in format HH:MM:SS to number of seconds.

    :return: duration in seconds
    :rtype: int
    """
    words = hhmmss.split(':')
    assert len(words)==3
    seconds = int(words[2]) + 60*( int(words[1]) + 60*int(words[0]) )
    return seconds
#-------------------------------------------------------------------------------

#===============================================================================
class Job:
    """
    class for storing and manipulating all data (:class:`JobSample` objects)
    related to a job.

    :param str timestamp: the timestamp of the first sample of the job.
    :param ShowqJobEntry job_entry: a job entry with the showq information of a job from a sample.
    :param Sampler sampler: (a reference to) the :class:`Sampler` object in charge.

    Occasionally, we need to examine other jobs to judge performance of a job, e.g.
    when a job is not using all resources of the node. For this reason :class:`Job`
    objects store a reference to the :class:`Sampler` object.
    """
    #---------------------------------------------------------------------------
    def __init__(self,timestamp,job_entry,sampler):
        assert isinstance(job_entry,ShowqJobEntry)
        self.jobid    = job_entry.get_jobid()
        self.username = job_entry.get_username()
        self.mhost    = job_entry.get_mhost()
        self.address  = None

        self.sampler = sampler

        self.nsamples_with_warnings = 0
        self.warning_counts = len(rules.the_rules)*[0]

        self.samples = OrderedDict() #{timestamp:JobSamnple object}
        self.first_timestamp = timestamp
        self.last_timestamp  = None
        self.jobscript       = None

        self.add_sample(job_entry,timestamp)
    #---------------------------------------------------------------------------
    def __str__(self):
        s = self.jobid    + '\n'
        s+= self.username + '\n'
        s+= self.mhost    + '\n'
        s+= str(self.samples)
        return s
    #---------------------------------------------------------------------------
    def add_sample(self,job_entry,timestamp):
        """
        Create a sample with the current *timestamp* from *job_entry*, and add it to the current Job.
        """
        self.last_timestamp = timestamp
        self.samples[timestamp] = JobSample(job_entry,self,timestamp)
    #---------------------------------------------------------------------------
    def timestamps(self):
        """
        :return: an ordered list of timestamps in this Job.
        """
        keys = list(self.samples.keys())
        return keys
    #---------------------------------------------------------------------------
#     def index(self,timestamp):
#         """
#         :return: the index of a timestamp.
#         """
#         index = self.timestamps().index(timestamp)
#         return index
    #---------------------------------------------------------------------------
    def nsamples(self):
        """
        :return: the number of samples in this Job, so far.
        """
        return len(self.samples)
    #---------------------------------------------------------------------------
    def check_for_issues(self,timestamp):
        """
        Verify wheter this job violates any of the rules for well-performing jobs.

        :return: an overview line if not all rules are satisfied, empty *str* otherwise
        """
        sample = self.samples[timestamp]
        if sample.check_for_issues():
            #there are issues
            if self.jobscript is None:
                self.jobscript = Data_jobscript(self.jobid,self.mhost)
            overview_line = sample.compose_overview()
        else:
            overview_line = ''
        return overview_line
    #---------------------------------------------------------------------------
    def overall_memory_used(self):
        """
        :return: the maximum amount of memory used by this Job, over all its samples.
        """
        mem_used = 0
        for sample in self.samples.values():
            mem_used = max(mem_used,sample.data_qstat.get_mem_used())
        return mem_used
    #---------------------------------------------------------------------------
    def get_details(self,timestamp):
        """
        :return the details text for *timestamp*.
        """
        if not timestamp in self.samples:
            timestamp = self.timestamps()[-1]
        details = self.samples[timestamp].compose_details()
        return details
    #---------------------------------------------------------------------------
    def remove_file(self):
        """
        Remove the *.pickled.gz* file corresponding to this Job.
        """
        fname = 'running/{}_{}.pickled.gz'.format(self.username,self.jobid)
        try:
            os.remove(fname)
        except:
            remote.err_print('failed to remove',fname)
    #---------------------------------------------------------------------------
    def get_sample(self,timestamp='last'):
        """
        Return the sample corresponding to *timestamp*, or, if it is *None*, the last sample.
        """
        if timestamp=='last':
            sample = od_last(self.samples)[1]
        else:
            sample = self.samples[timestamp]
        return sample
    #---------------------------------------------------------------------------
    def get_nnodes(self,timestamp='last'):
        """
        :return: the numer of nodes as reported by the sample at *timestap*
        """
        sample = self.get_sample(timestamp)
        return sample.get_nnodes()
    #---------------------------------------------------------------------------
    def get_ncores(self,timestamp='last'):
        """
        :return: the numer of nodes as reported by the sample at *timestap*
        """
        sample = self.get_sample(timestamp)
        return sample.get_ncores()
    #---------------------------------------------------------------------------
    def get_mem(self,timestamp='last'):
        """
        :return: the maximum of memory used and requested for the given timestamp.
        """
        sample = self.get_sample(timestamp)
        return sample.get_mem()
    #---------------------------------------------------------------------------
    def pickle(self,prefix,only_if_warnings=True,verbose=False,compressed=True):
        """
        Pickle this job and compress it using gzip.

        :param str prefix: the receiving directory.
        :param bool only_if_warnings: do only pickle if the job has warnings.
        :param bool verbose: if *True*, print the destination file.
        :param bool compressed: compress after pickling.
        """
        if (only_if_warnings and self.nsamples_with_warnings) \
        or (not only_if_warnings):
            #pickle
            if 'running' in prefix:
                fname = '{}_{}.pickled'   .format(self.username,self.jobid)
            else:
                fname = '{}_{}_{}.pickled'.format(self.username,self.jobid,self.timestamps()[-1])

            if compressed:
                fpath = os.path.join(prefix,fname+'.gz')
                fo = gzip.open(fpath,'wb')
            else:
                fo =      open(fpath,'wb')
            # remove the "upward" object references in the data tree
            # otherwise they waste a lot of disk space
            sampler = self.sampler
            self.sampler = None
            # job_sample.parent_job
            for job_sample in self.samples.values():
                job_sample.parent_job = None
            # pickle this job
            pickle.dump(self,fo)
            # finally restore the upward references
            self.sampler = sampler
            # job_sample.parent_job
            for job_sample in self.samples.values():
                job_sample.parent_job = self

            if verbose:
                print(' (pickled {})'.format(fpath))
            fo.close()
    #---------------------------------------------------------------------------

#===============================================================================
def unpickle(fpath,sampler=None,verbose=False):
    """
    Counterpart of Job.pickle()

    :param str fpath: path to pickled file. If ending on '.pickled.gz', unzips before unpickling. If ending on '.pickled', unpickle without unzipping. Otherwise try both in that order.
    :param Sampler sampler: :class:`Sampler` object or :class:`None`.
    :param bool verbose: print the filename of the unpickled file.
    :return: a Job object or :class:`None` if the file does not exist.
    """
    if fpath.endswith('.pickled.gz'):
        try:
            with gzip.open(fpath,'rb') as fo:
                job = pickle.load(fo)
        except:
            job = None

    elif fpath.endswith('.pickled'):
        try:
            with open(fpath,'rb') as fo:
                job = pickle.load(fo)
        except:
            job = None
    else:
        _fpath = fpath+'.pickled.gz'
        if os.path.exists(_fpath):
            return unpickle(_fpath, sampler=sampler, verbose=verbose)
        else:
            _fpath = fpath+'.pickled'
            if os.path.exists(_fpath):
                return unpickle(_fpath, sampler=sampler, verbose=verbose)
            else:
                job = None
    if job is None:
        if verbose:
            print(' (not found {})\n'.format(fpath))
    else:
        if verbose:
            print(' (unpickled {})\n'.format(fpath))
        # Set the "upward" object references in the data tree
        # otherwise they refer to their version at the moment of pickling
        job.sampler = sampler
        # job_sample.parent_job
        for job_sample in job.samples.values():
            job_sample.parent_job = job
    return job
#===============================================================================
class Sampler:
    """
    Class that does the sampling, either through :func:`sample` or :func:`fetch_offline_samples`.
    The frequency of sampling is determined by the caller.

    :param int interval: number of seconds between successive samples.
    :param qMainWindow: If *None* prints a progress bar to the terminal during sampling, otherwise uses a Qt4:QProgressDialog.
    """
    #---------------------------------------------------------------------------
    def __init__(self,interval=None,qMainWindow=None):
        if interval is None:
            self.sampling_interval = Cfg.sampling_interval
        else:
            self.sampling_interval = interval
        self.qMainWindow = qMainWindow

        self.overviews = OrderedDict()      # {timestamp:job_overview}
        self.jobs    = {}                   # {jobid    :Job object  }
        self.timestamps = []                # [datetime.strftime(timestamp_format)]
        self.timestamp_jobs = OrderedDict() # {timestamp:[jobids]}
        self.jobids_running_previous = []
    #---------------------------------------------------------------------------
    def sample(self,verbose=False,show_progress=False):
        """
        Sample the running jobs online (locally).
        """
        self.data_showq = remote.run("showq -r -p hopper --xml",post_processor=remote.xml_to_odict)
        self.total_nodes_in_use = self.get_total_nodes_in_use()
        # get the job entries
        try:
            job_entries = self.data_showq['Data']['queue']['job']
        except:
            remote.err_print('No jobs running.')
            exit(0)
        # remove jobs
        #  . which have no mhost set
        #  . which have jobid like '390326[1]' (=job array jobs)
        job_entries_filtered = []
        for job_entry in job_entries:
            converted = ShowqJobEntry(job_entry)

            # ignore jobs with unknow mhost
            try:
                converted.get_mhost()
            except KeyError:
                print('ignoring',converted.get_jobid_long(), '(mhost unknown)')
                continue

            # ignore jobs with jobid containing '[n]'
            jobid = converted.get_jobid()
            if '[' in jobid:
                print('ignoring',job_entry.get_jobid_long(), '(worker job)')
                continue
            job_entries_filtered.append(converted)
        job_entries = job_entries_filtered

        self.n_entries   = len(job_entries)

        if self.qMainWindow:
            from PyQt4.QtGui import QProgressDialog,QApplication
            dlg = QProgressDialog('','',0, self.n_entries,self.qMainWindow)
            hdr = 'Sampling #{} : {} {}/{}'
        else:
            if show_progress:
                from progress import printProgress
                hdr = 'sampling showq #{}'.format(len(self.timestamp_jobs)+1)

        # create
        #   . a dict { mhost : [jobid] } with all the jobs running on node mhost
        #   . a list wit all uncompleted jobids
        # the latter is compared to the jobid list of the previous sample to find
        # out which jobs are finished.
        self.mhost_jobs = OrderedDict()
        self.jobids_running = []
        for job_entry in job_entries:
            mhost = job_entry.get_mhost()
            jobid = job_entry.get_jobid()
            od_add_list_item(self.mhost_jobs,mhost,jobid)
            self.jobids_running.append(jobid)
            try:
                self.jobids_running_previous.remove(jobid)
            except ValueError:
                pass
            #   when this loop has completed, self.jobids_running_previous
            #   contains only jobides of finished jobs.
        jobids_finished = self.jobids_running_previous
        self.jobids_running_previous = self.jobids_running # prepare for next sample() call
        # pickle finished jobs (if they had issues) and remove them from self.jobs
        os.makedirs('completed', exist_ok=True)
        for jobid in jobids_finished:
            try:
                job = self.jobs.pop(jobid)
            except KeyError:
                continue
            job.pickle('completed/',verbose=True)
            if Cfg.offline:
                job.remove_file()
        timestamp = get_timestamp_now()
        if Cfg.offline:
            os.makedirs ('running',exist_ok=True)
            if os.path.exists('running/timestamp'):
                os.remove('running/timestamp')
            #   if ths file is absent ojm is sampling.
            print(title_line(timestamp, char='=', width=100, above=True, below=True),end='')

        # loop over the running jobs (job_entries)
        #pass 1 create jobs and job samples
        for i_entry,job_entry in enumerate(job_entries):
            if job_entry.get_state() != 'Running':
                continue # we only analyze running jobs
            jobid    = job_entry.get_jobid()
            #username = job_entry.get_username()
            od_add_list_item(self.timestamp_jobs,timestamp,jobid)

            if self.qMainWindow:
                progress_message = hdr.format(len(self.timestamp_jobs),jobid,i_entry,self.n_entries)
                dlg.setLabelText(progress_message)
                dlg.setValue(i_entry)
                QApplication.processEvents()
            else:
                if show_progress:
                    printProgress(i_entry, self.n_entries, prefix=hdr, suffix='jobid='+jobid, decimals=-1)

            job = self.jobs.get(jobid,None)
            if job is None:
                # this job is encountered for the first time
                job = Job(timestamp,job_entry,self)
                self.jobs[jobid] = job
            else:
                job.add_sample(job_entry,timestamp)
                if job.sampler is None:
                    remote.err_print('### strange')
                    job.sampler = self

        if self.qMainWindow:
            # terminate QProgressDialog
            dlg.setValue(self.n_entries)
            QApplication.processEvents()
            # start new QProgressDialog
            dlg = QProgressDialog('','',0, self.n_entries,self.qMainWindow)
            hdr = 'Checking rules #{} : {} {}/{}'
        else:
            if show_progress:
                # terminate printProgress
                printProgress(self.n_entries, self.n_entries, prefix=hdr, suffix='', decimals=-1)
                # start new printProgress
                hdr = 'Checking rules #{}'.format(len(self.timestamp_jobs)+1)

        #pass 2 add NeighbouringJobInfo and check the rules
        overview = [] # one warning per job with issues, jobs without issues are skipped
        i_entry = 0
        for jobid,job in self.jobs.items():
            #progress
            if self.qMainWindow:
                progress_message = hdr.format(len(self.timestamp_jobs),jobid,i_entry,self.n_entries)
                dlg.setLabelText(progress_message)
                dlg.setValue(i_entry)
                QApplication.processEvents()
            else:
                if show_progress:
                    printProgress(i_entry, self.n_entries, prefix=hdr, suffix='jobid='+jobid, decimals=-1)
            i_entry += 1
            #the real work
            overview_line = job.check_for_issues(timestamp)
            if overview_line:
                overview.append(overview_line)
                if verbose:
                    print('\n'+timestamp+'\n')
                    print(job.get_details(timestamp))
                if Cfg.offline:
                    job.pickle('running', verbose=verbose)

        if self.qMainWindow:
            # terminate QProgressDialog
            dlg.setValue(self.n_entries)
            QApplication.processEvents()
        else:
            if show_progress:
                # terminate printProgress
                printProgress(self.n_entries, self.n_entries, prefix=hdr, suffix='', decimals=-1)
            for line in overview:
                print(line,end='')
            print('\nWell performing jobs: {}/{}'.format(self.n_entries-len(overview),self.n_entries))
        print(self.total_nodes_in_use)

        if Cfg.offline:
            # notify that sampling has finished..
            with open('running/timestamp','w') as f:
                f.write(timestamp)

        self.overviews[timestamp] = self.overview_list2str(overview)

        self.timestamps.append(timestamp)
        #    this must be the last statement because the gui otherwise sees a timestamp which is not ready.
        return timestamp
    #---------------------------------------------------------------------------
    def get_total_nodes_in_use(self):
        """
        :return: a str describing the fraction of nodes in use.
        """
        if hasattr(self,'data_showq'):
            s = 'nodes in use: {}/{}'.format( self.data_showq['Data']['cluster']['@LocalActiveNodes']
                                            , self.data_showq['Data']['cluster']['@LocalConfigNodes'] )
            self.total_nodes_in_use = s
            return s
        else:
            if hasattr(self,'total_nodes_in_use'):
                return self.total_nodes_in_use
            else:
                return ''
    #---------------------------------------------------------------------------
    def get_remote_timestamp(self):
        """
        :return: the last sample's timestamp from the offline job monitor. If ojm.py is in the process of sampling an empty string is returned.
        """
        try:
            lines = remote.run('cd data/jobmonitor/running/; cat timestamp',post_processor=remote.list_of_lines)
            return lines[0]
        except:
            return ''
    #---------------------------------------------------------------------------
    def fetch_offline_samples(self):
        """
        Sample the running jobs from the offline job monitor. The remote directory '~/data/jobmonitor/running'
        examined to see if there are new samples available. These are copied to the local directory ./offline/running
        """
        shutil.rmtree('offline/running')
        os.makedirs('offline/running'  ,exist_ok=True)
        #os.makedirs('offline/completed',exist_ok=True)
        timestamp = self.get_remote_timestamp()
        while not timestamp:
            sleep(60)
            timestamp = self.get_remote_timestamp()
        if self.timestamps:
            if timestamp==self.timestamps[-1]:
                return # this timestamp is already in the samples
        self.timestamps.append(timestamp)
        filenames = remote.glob('*.pickled.gz','data/jobmonitor/running/')
        self.n_entries = 0
        for filename in filenames:
            if not filename: # empty line
                continue
            lfname =         'offline/running/'+filename
            rfname = 'data/jobmonitor/running/'+filename
            print('copying '+rfname,'to',lfname,end='')
            try:
                remote.copy_remote_to_local(lfname,rfname)
                print(' - copied')
            except:
                print(' - failed')
                continue
            job = unpickle('offline/running/'+filename,sampler=self)
            self.add_offline_job(job)
            self.n_entries += 1
        self.overviews[timestamp] = self.overview_list2str(self.overviews[timestamp])
    #---------------------------------------------------------------------------
#     def timestamp(self,i=-1):
#         return self.timestamps[i]
    #---------------------------------------------------------------------------
    def overview_list2str(self,overview_list,key=overview_by_user,reverse=True):
        """
        Sort the *overview_list* according to *key* and *reverse* and convert to plain text.
        """
        overview_list.sort(key=key,reverse=reverse)
        n_jobs = self.n_entries
        n_warn = len(overview_list)
        text = 'Jobs running well: {}/{}, efficiency threshold = {}%'.format(n_jobs-n_warn,n_jobs,Cfg.effic_threshold)
        s = self.get_total_nodes_in_use()
        if s:
            text+= ', '+s
        text+= ''.join(overview_list)
        return text
    #---------------------------------------------------------------------------
    def nsamples(self):
        """
        :return: the number of samples.
        """
        return len(self.timestamps)
    #---------------------------------------------------------------------------
    def add_offline_job(self,job):
        """
        Add an offline monitored *job* to the sampler.
        """
        self.jobs[job.jobid] = job
        for timestamp,job_sample in job.samples.items():
            od_add_list_item(self.timestamp_jobs,timestamp,job.jobid)
            overview_line = job_sample.compose_overview()
            if not timestamp in self.overviews:
                self.overviews[timestamp] = [overview_line]
            else:
                overview = self.overviews[timestamp]
                if isinstance(overview,str):
                    lines = overview.split('\n')
                    self.overviews[timestamp] = []
                    overview = self.overviews[timestamp]
                    for line in lines:
                        if not line.endswith('\n'):
                            line += '\n'
                        overview.append(line)
                overview.append(overview_line)
        self.total_nodes_in_use = job.sampler.get_total_nodes_in_use()
    #---------------------------------------------------------------------------
    def when_done_adding_offline_jobs(self):
        """
        Sort offline monitored jobs.
        """
        self.timestamps.sort()
        for jobid_list in self.timestamp_jobs.values():
            jobid_list.sort()
    #---------------------------------------------------------------------------

################################################################################
# test code below
################################################################################
if __name__=="__main__":
    remote.connect_to_login_node()

    sampler = Sampler()
    timestamp = sampler.sample()

    current_jobids = sampler.timestamp_jobs[timestamp]
    for jobid in current_jobids:
        job_sample = sampler.jobs[jobid].samples[timestamp]
        print( job_sample.compose_details(),end='')

    print('\n--finished--')

{'QProgressDialog': 'PyQt4.QtGui.QProgressDialog', 'QApplication': 'PyQt4.QtGui.QApplication', 'printProgress': 'progress.printProgress'}


import itertools
from operator import truediv
import subprocess
import time
import timeit
from unittest.mock import patch
from Cython.Compiler.TreeFragment import *
from Cython.Compiler.Visitor import *
from Cython.Compiler.Main import *
from Cython.CodeWriter import *
from Cython.Compiler.AnalysedTreeTransforms import *
from Cython.Compiler.ParseTreeTransforms import *
from Cython.Compiler.Symtab import *
from algorithm.algorithm import Algorithm
import utilities.cython_tree as cy
from edit import TypeInsertion
from patch import Patch
from examples import fib

class ExhaustiveSearch(Algorithm):
    def __init__(self, program):
        self.program = program

    def run(self):
        run_start = time.time()
        normal = NormalizeTree(None)
        types = ['short', 'long']
        ast = cy.file_to_ast(self.program.file_path)
        normal.visit(ast)
        modifications = len(self.program.modification_points)
        points = [x for x in range(modifications)]

        patches = []

        combinations = [p for p in itertools.product(
            types, repeat=modifications)]

        for c in combinations:
            zipped = zip(c, points)
            patches.append(list(zipped))
        best_patch = Patch(self.program)

        print(patches)

        start = timeit.default_timer()
        expected = fib.fib(75)
        end = timeit.default_timer()
        fastest = end-start
        fast_index = 0
        print(fastest)

        file = 'output/temp.pyx'
        result_list = {"success": 0, "compilation_fail": 0,
                       "bug_found": 0, "wrong_value": 0}

        for i in range(len(patches)):
            print(i)
            new_patch = best_patch.clone()
            for edit in patches[i]:
                new_patch.add(TypeInsertion.create(
                    self.program, edit[1], edit[0]))
            result = self.program.evaluate_patch(new_patch)
            print(result)

            result_list[result["status"]] += 1

            if result["status"] == "success":
                value = float(result["value"])
                if value < fastest:
                    fastest = value
                    fast_index = i
                    best_patch = new_patch

        print(fast_index)
        print(fastest)
        print(result_list)
        self.program.write_result(best_patch)

        run_end = time.time()
        print(run_end-run_start)

        return


import sqlite3

con =sqlite3.connect("sample.db")
cursor = con.cursor()

# User名を指定して、Aeg （年齢）を更新するprogram

# ユーザーに何をしてもらいたいか？
target_name = input("whose 'age' do you want to update?")
new_age = input("Tell me new age:")
print(target_name, new_age)

# SQL 文を作成
# 文字列を使って SQL を作成するという事は脆弱性があるので注意する
# SQL ingection という攻撃方法 : new age の後にセミコロン「; DROP TABLE User;」 で User table data を消されてしまう
# update_query = f"UPDATE User SET age = {new_age} WHERE name = '{target_name}'"
# cursor.executescript(update_query)

# User の input で update する。ほとんどそのような場合だが…
# 上記のような攻撃を防ぐ為に必ず、 execute method を使って プレイスフォルダーを使って値を tuple で渡す
update_query = 'UPDATE User SET age = ? WHERE name = ?'
cursor.execute(update_query, (new_age, target_name))
con.commit()


import os
from setuptools import setup

DESC = "Some decorators and classes to make working with django projects easier."

try:
    with open('readme.rst') as file:
        long_description = file.read()
except IOError:
    long_description = DESC

setup(
    name = "django-drapes",
    version = "0.1.2",
    author = "Ulas Tuerkmen",
    author_email = "afroisalreadyinu@gmail.com",
    description = DESC,
    long_description=long_description,
    install_requires = ['formencode',
                        'decorator',
                        'django',
                        'coverage',
                        'nose',
                        'mock'],
    py_modules=['django_drapes'],
    classifiers=['Framework :: Django',],
)


from django.urls import path
from django.contrib.auth.decorators import login_required
from apps.citas.views import *
urlpatterns =[
    path('listar_tipo_mascota/',login_required(TipoMascotaListView.as_view()), name='listar_tipo_mascota'),
    path('tipo_mascota/',login_required(TipoMascotaView.as_view()), name='tipo_mascota'),
    path('modificar_tipo_mascota/',login_required(TipoMascotaUpdateView.as_view()), name='modificar_tipo_mascota'),
    path('eliminar_tipo_mascota/<int:id>/',login_required(eliminarTipoMascota), name='eliminar_tipo_mascota'),

    path('listar_raza/',login_required(RazaListView.as_view()), name='listar_raza'),
    path('raza/',login_required(RazaView.as_view()), name='raza'),
    path('modificar_raza/',login_required(RazaUpdateView.as_view()), name='modificar_raza'),
    path('eliminar_raza/<int:id>/',login_required(eliminarRaza), name='eliminar_raza'),

    path('listar_mascota/',login_required(MascotaListView.as_view()), name='listar_mascota'),
    path('mascota/',login_required(MascotaView.as_view()), name='mascota'),
    path('modificar_mascota/',login_required(MascotaUpdateView.as_view()), name='modificar_mascota'),
    path('eliminar_mascota/<int:id>/',login_required(eliminarMascota), name='eliminar_mascota'),
    path('buscar_cliente/<str:dni>/',login_required(buscarCliente), name='buscar_cliente'),
    path('listar_tipo_servicio/',login_required(TipoServicioListView.as_view()), name='listar_tipo_servicio'),
    path('tipo_servicio/',login_required(TipoServicioView.as_view()), name='tipo_servicio'),
    path('modificar_tipo_servicio/',login_required(TipoServicioUpdateView.as_view()), name='modificar_tipo_servicio'),
    path('eliminar_tipo_servicio/<int:id>/',login_required(eliminarTipoServicio), name='eliminar_tipo_servicio'),

    path('listar_servicio/',login_required(ServicioListView.as_view()), name='listar_servicio'),
    path('servicio/',login_required(ServicioView.as_view()), name='servicio'),
    path('modificar_servicio/',login_required(ServicioUpdateView.as_view()), name='modificar_servicio'),
    path('eliminar_servicio/<int:id>/',login_required(eliminarServicio), name='eliminar_servicio'),

    #Cita
    #path('listar_cita/',login_required(CitaListView.as_view()), name='listar_cita'),
    #path('cita/', login_required(CitaView.as_view()), name='cita'),
    #path('modificar_cita/',login_required(MascotaUpdateView.as_view()), name='modificar_cita'),
    #path('eliminar_cita/<int:id>/',login_required(eliminarCita), name='eliminar_cita'),

]


from django.db.models import Avg, Count
from django.shortcuts import get_object_or_404
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import permissions
from rest_framework.pagination import LimitOffsetPagination
from rest_framework.viewsets import ModelViewSet

from reviews.models import Category, Genre, Review, Title
from .filters import TitleFilter
from .mixins import CreateListDestroyViewSet
from .permissions import (AnonimReadOnly,
                          IsSuperUserIsAdminIsModeratorIsAuthor,
                          IsSuperUserOrIsAdminOnly, )
from .serializers import (CategorySerializer, CommentSerializer,
                          GenreSerializer, ReviewSerializer,
                          TitleNotSafeSerializer, TitleSafeSerializer)

class CategoryViewSet(CreateListDestroyViewSet):
    """Вьюсет для создания обьектов класса Category."""

    queryset = Category.objects.all()
    serializer_class = CategorySerializer
    lookup_field = 'slug'

class GenreViewSet(CreateListDestroyViewSet):
    """Вьюсет для создания обьектов класса Genre."""

    queryset = Genre.objects.all()
    serializer_class = GenreSerializer
    lookup_field = 'slug'

class TitleViewSet(ModelViewSet):
    """
    Получить список всех объектов. Права доступа: Доступно без токена
    """
    queryset = Title.objects.annotate(rating=Avg('reviews__score'))
    permission_classes = (AnonimReadOnly | IsSuperUserOrIsAdminOnly,)
    filter_backends = (DjangoFilterBackend,)
    filterset_class = TitleFilter
    pagination_class = LimitOffsetPagination

    def get_serializer_class(self):
        if self.action in ('list', 'retrieve'):
            return TitleSafeSerializer
        return TitleNotSafeSerializer

class ReviewViewSet(ModelViewSet):
    serializer_class = ReviewSerializer
    permission_classes = (
        permissions.IsAuthenticatedOrReadOnly,
        IsSuperUserIsAdminIsModeratorIsAuthor,
    )
    pagination_class = LimitOffsetPagination

    def get_title(self):
        title_id = self.kwargs.get('title_id')
        return get_object_or_404(Title, pk=title_id)

    def get_queryset(self):
        return self.get_title().reviews.all()

    def perform_create(self, serializer):
        serializer.save(
            author=self.request.user,
            title=self.get_title()
        )

class CommentViewSet(ModelViewSet):
    serializer_class = CommentSerializer
    permission_classes = (
        permissions.IsAuthenticatedOrReadOnly,
        IsSuperUserIsAdminIsModeratorIsAuthor,
    )
    pagination_class = LimitOffsetPagination

    def get_review(self):
        review_id = self.kwargs.get('review_id')
        comments = Review.objects.annotate(count=Count("comments"))
        return get_object_or_404(comments, pk=review_id,)

    def get_queryset(self):
        return self.get_review().comments.all()

    def perform_create(self, serializer):
        serializer.save(
            author=self.request.user,
            review=self.get_review())