import random

# functions go here...
def yes_no(question):
	valid = False
	while not valid:
		response = input(question).lower()

		if response == "yes" or response == "y":
				response = "yes"
				return response

		elif response == "no" or response == "n":
				response = "no"
				return response

		else:
			print("please answer yes/no")

def instructions():
	print ("**** How to Play ****")
	print()
	print("in this paticular addition game the aim is to answer as many addition questions as you can correctly.")
	print ()
	return ""

# Main Routine goes here...
played_before = yes_no("Have you played the game before? ")

if played_before == "no":
	instructions()

elif played_before == "yes":
	print("program continues")
def num_check(question, low, high):
	error = "Please enter a whole number between low and high\n"

	valid = False
	while not valid:
		try:
			# Ask the question
			response = int(input(question))
			# if the amount is too low / too high give
			if low < response <= high:
				return response

			# output an error
			else:
				print(error)

		except ValueError:
				print(error)

# Main routine go here
print("**** This is a 10 Question addition quizz ****")
low_num = num_check("What is the lowest number for a question? ", -500, 500)
high_num = num_check("What is the highest number for a question? ", low_num, 1000)

# Rounds won will be calculated (total - correct - incorrect)
questions_correct = 0
questions_incorrect = 0

game_summary = []

rounds = 10

	# Start of Game Play Loop

	# Rounds Heading for continuous mode / specific # of rounds
print()
if rounds == "":
	heading = "continuous Mode: Round {}".format(questions_correct + 1)
else:
 heading = "Question 1 of 10".format(questions_incorrect + 1)

print(heading)
for item in range(0, 10):

	num_1 = random.randint(1, 10)
	num_2 = random.randint(1, 10)

	print("*Question*")
	question = "{} + {}".format(num_1, num_2)
	answer = eval(question)

	user_ans = int(input("What is {} = ".format(question)))

	if user_ans == answer:
		print("Correct!, Well Done :)")
		questions_correct += 1

	else:
		print("incorrect, Nice Try :(")
		questions_incorrect += 1

print("Game Over!")
print("*** Here are your results ***")

feedback ="questions correct: {} questions incorrect: {}".format(questions_correct,questions_incorrect)
print(feedback)
print()

# add result to list....
game_summary.append(feedback)


from aot.meta_triggers.DisplayInstructions import DisplayInstructions, InstructionParameters
from aot.meta_triggers.invicible_unit import InvicibleUnit
from aot.meta_triggers.no_house_needed import NoHouseNeeded
from aot.model.condition import *
from aot.model.effect import *
from aot.model.unit import Unit
from aot.model.enums.player import PlayerEnum
from aot.model.enums.sizes import Size
from aot.model.enums.tile import EnumTile
from aot.model.enums.unit import UnitConstant, UnitType
from aot.model.scenario import Scenario
from aot.model.trigger import Trigger
from aot.utilities.configuration import Configuration
import logging
import numpy as np
import random

logging.basicConfig(level=logging.INFO)
C = Configuration("examples/configuration_de.json")

scn = Scenario(size=Size.TINY)
scn.load_template(Size.TINY)

#Team
class Team:
    def __init__(self, name):
        self.name = name

        self.x1_win = None
        self.x2_win = None
        self.y1_win = None
        self.y2_win = None
        self.players = None
        self.other_team = None
        self.players_starts = [None] * 4

class Player:

    def __init__(self, id):
        self.id = id
        self.position = (None, None)

team1 = Team("TEAM 1")
team1.players = [Player(i) for i in range(1, 5)]

#Diplomacy
diplomacy = Trigger("Set Diplomacy", enable=1).then_(
    SendInstruction(message="Setting Diplomacy 1234",
                    color=Color.ORANGE,
                    panel_location=1, time=5))

for team in [team1]:
    for player in team.players:
        for target_player in team.players:
            diplomacy.then_(ChangeDiplomacy(source_player=player.id, target_player=target_player.id, diplomacy=0))

scn.add(diplomacy)

#Merchant
merchants = []
for team in [team1]:
    for player in team.players:
        merchant = scn.units.new(owner=player.id, x=5, y=5, unit_cons=UnitConstant.MERCHANT.value)
        merchant.reset_position = (5, 5)
        merchants.append(merchant)

#Fire
for i in range(1, 450):
    randx = random.randint(3, scn.get_width() / 3 - 1)
    randy = random.randint(3, scn.get_height() / 3 - 1)
    smoke = scn.units.new(owner=PlayerEnum.GAIA.value, x=randx+2, y=randy-1, unit_cons=UnitConstant.SMOKE.value)
    for team in [team1]:
        for player in team.players:
            fire = Trigger("Fire touching {}".format(player.id), enable=1, loop=True)
            fire \
            .if_(ObjectInArea(amount=1, source_player=player.id, unit_cons=UnitConstant.MERCHANT.value, x1=randx, x2=randx, y1=randy, y2=randy)) \
            .then_(RemoveObjectByConstant(player=player.id, unit_cons=UnitConstant.MERCHANT.value, x1=randx, x2=randx, y1=randy, y2=randy)) \
            .then_(CreateObject(player=player.id, unit_cons=UnitConstant.MERCHANT.value, x=3, y=3))
            scn.add(fire)

#Script
OFFSET_TUTO = 5
for team in [team1]:
    for player in team.players:
        offset = 0
        m1 = InstructionParameters("Welcome to Age of Total War", None, 2 * OFFSET_TUTO, Color.BLUE)
        m2 = InstructionParameters("To win a round, you need to bring 3 relics cart to this area", None, 2 * OFFSET_TUTO, Color.BLUE)
        m3 = InstructionParameters("test", None, 2 * OFFSET_TUTO, Color.BLUE)
        m4 = InstructionParameters("no", None, 2 * OFFSET_TUTO, Color.BLUE)
        messages = [m1, m2, m3, m4]
        display_tutorial = DisplayInstructions(messages=messages, name="()".format(player.id), enable=True, player=player.id, panel_location=2)
        scn.add(display_tutorial)

scn.save(C.game_path_scenario, "mygame")


import json
import pandas as pd
import numpy as np

from enum import Enum
from typing import List
from dataset_attributes import DataSetAtrributes
from sklearn.impute import SimpleImputer

import warnings

warnings.filterwarnings('ignore')

class ImputerStrategy(Enum):
    MEAN = 'mean'
    MEDIAN = 'median'
    MODE = 'mode'
    CONSTANT = 'constant'
    REGRESSOR_MODEL = 'regressor_model'
    CLASSIFICATION_MODEL = 'clasification_model'
    MOST_FREQUENT = 'most_frequent'

class DataBot:
    numeric_types: List[str] = ['int64', 'float64', 'datetime64']
    string_types: List[str] = ['object', 'category']

    def __init__(self,
                 dataset=None,
                 target_name=None,
                 null_threshold=0.3,
                 cardinal_threshold=0.3,
                 project_path=None):
        self.dataset = dataset
        self.target = target_name
        self.null_threshold = null_threshold
        self.cardinal_threshold = cardinal_threshold
        self.project_path = project_path
        self.categorical_columns = []
        self.numeric_columns = []

        self.datasetAttributes = DataSetAtrributes(self.project_path)

        if target_name is not None:
            self.target_name = target_name
            self.target = self.dataset[self.target_name]
            self.features = self.dataset.drop([self.target_name], axis=1)
        else:
            self.features = dataset

    # Lambda for a series object that fill nulls with the mean.
    fill_mean = None
    # Lambda for a series object that fill nulls with the mode.
    fill_mode = None

    def scale_range(self, x):
        return (x - x.min(axis=0)) / (x.max(axis=0) - x.min(axis=0))

    def scale_log(self, x):
        return np.log(x + 1)

    impute_strategies = {
        ImputerStrategy.MEAN: fill_mean,
        ImputerStrategy.MODE: fill_mode
    }

    def impute(self, columns, impute_strategy):
        """Impute selected columns (pd.Series) from self.features with the given strategy.

        Parameters
        ----------
        :param columns: list of columns names to impute.
        :param impute_strategy: Selected ImputerStrategy
        """
        print("Column",columns ,"Before", self.features[columns])
        imp = SimpleImputer(missing_values=np.nan, strategy=impute_strategy.value)
        imp = imp.fit(self.features[columns])
        self.features[columns] = imp.transform(self.features[columns])
        print("Column",columns, "After", self.features[columns])

    def one_hot_encode(self, col_name, categorical_values):
        """ Apply one hot encoding to the given column.

        :param col_name: Name of the column to one hot encode.
        :param categorical_values: Unique values from self.features[col_name]
        :return:
        """

        # Get one hot encoding of column
        # one_hot = pd.get_dummies(data=categorical_values, columns=col_name)
        one_hot = pd.get_dummies(self.features[col_name], columns=col_name)
        print("one hot", categorical_values, one_hot)
        # Drop column as it is now encoded
        self.features = self.features.drop(col_name, axis=1)
        # Join the encoded df
        self.features = self.features.join(one_hot)
        print("one hot after", categorical_values, self.features)

    def normalize(self, columns):
        """Apply self.scale_range and self.scale_log to the given columns
        :param columns: list of columns names to normalize
        """
        self.features[columns] = self.features[columns].apply(lambda x: self.scale_range(x))
        self.features[columns] = self.features[columns].apply(lambda x: self.scale_log(x))

    def remove_null_columns(self):
        """Remove columns with a percentage of null values greater than the given threshold (self.null_threshold).

        """
        threshold = self.null_threshold * self.row_count()
        columns = self.features.dropna(axis='columns', thresh=threshold, inplace=True)
        # self.datasetAttributes.parameters['removed_columns'] += columns
        self.datasetAttributes.parameters['removed_columns'] += self.dataset.columns.difference(
            self.features.columns).tolist()

    def remove_high_cardinality_columns(self):
        """Remove columns with a cardinality percentage greater than the given threshold (self.cardinal_threshold).

        """
        threshold = self.null_threshold * self.row_count()
        columns_with_high_cardinality = [col for col in self.features.columns if self.features[col].nunique() > threshold]
        columns = self.features.dropna(axis='columns', thresh=threshold, inplace=True)
        # self.datasetAttributes.parameters['removed_columns'] += columns
        self.datasetAttributes.parameters['removed_columns'] += self.dataset.columns.difference(self.features.columns).tolist()

    def row_count(self):
         return len(self.features.index)

    def pre_process(self):
        """Preprocess dataset features before being send to ML algorithm for training.
        """
        # Implement this method with the given indications in the given order

        # Remove columns with null values above the threshold
        self.remove_null_columns()

        # Remove columns with cardinality above the threshold
        self.remove_high_cardinality_columns()

        # Create a python list with the names of columns with numeric values.
        # Numeric columns have one of the types stored in the list self.numeric_types
        self.numeric_columns = list(self.features.select_dtypes(include=self.numeric_types).columns)

        # Create a python list with the names of columns with string values.
        # Categorical columns have one of the types stored in the list self.string_types
        self.categorical_columns = list(self.features.select_dtypes(include=self.string_types).columns)

        # Create a python list with the names of numeric columns with at least one null value.
        print("Numeric columns:", self.numeric_columns)
        print("Categorical columns:", self.categorical_columns)
        print(self.features.head(10))
        # numeric_nulls = self.features.loc[self.numeric_columns, self.features.isnull().any()].columns
        numeric_nulls = [col for col in self.numeric_columns if self.features[col].isnull().any()]

        # Create a python list with the names of categorical columns with at least one null value.
        categorical_nulls = [col for col in self.categorical_columns if self.features[col].isnull().any()]
        # categorical_nulls = self.features.loc[self.categorical_columns, self.features.isnull().any()].columns

        # Impute numerical columns with at least one null value.
        self.impute(numeric_nulls, impute_strategy=ImputerStrategy.MEAN)

        # Impute categorical columns with at least one null value.
        if categorical_nulls:
            self.impute(categorical_nulls, impute_strategy=ImputerStrategy.MOST_FREQUENT)

        # These two lines gather information from the dataset for further use.
        self.datasetAttributes.set_column_values(self.categorical_columns, self.features)
        self.datasetAttributes.set_number_values(self.numeric_columns, self.features)

        # Apply one hot encoding to all categorical columns.
        for col in self.categorical_columns:
            categorical_values = self.features[col].unique()
            self.one_hot_encode(col, categorical_values)

        # Normalize all numeric columns
        # TODO uncomment
        self.normalize(self.numeric_columns)

        # This line store relevant information from the processed dataset for further use.
        # TODO uncomment line below
        self.datasetAttributes.save()
        # print(self.features)
        self.features = self.features.reset_index()
        print(self.features.isnull())
        print("los nuuuuuuulos", self.features.isnull().any())

    def pre_process_prediction(self, parameters):
        """Preprocess records from API calls before running predictions

        :param parameters: information from the processed dataset in the training stage.

        """
        self.features.drop(parameters['removed_columns'], axis=1, inplace=True)

        for column in parameters['categorical_columns'].keys():
            categorical_values = parameters['categorical_columns'][column]["values"]
            self.one_hot_encode(column, categorical_values)

        for column in parameters['numeric_columns'].keys():
            n_min = parameters['numeric_columns'][column]["min"]
            n_max = parameters['numeric_columns'][column]["max"]
            self.features[column] = self.features[column].apply(lambda x: (x - n_min) / (n_max - n_min))
            self.features[column] = self.features[column].apply(lambda x: np.log(x + 1))

    def get_dataset(self):
        """Returns a dataset with features and labels.

        :return: Dataset with features and labels.
        """
        self.dataset = self.features
        self.dataset[self.target_name] = self.target
        return self.dataset


s = "python3"
r = "raja"
sh = 'shobha'
iv = 'Ivaan'
ag = "34"
python_course = True
r_course = False
number_of_sisters = None

# print("Hi ", r.capitalize(), s.capitalize(), sh.upper())
# print(sh.isalpha())
# print(ag.isdigit())

# print("Hello, am {0}. Nice to meet you {1}.".format(r.capitalize(),sh.capitalize()))

print(f"Hello, am {r}. Nice to meet you {sh} and {iv}. Learning Python.. Is that {python_course}?. Ivaan has {number_of_sisters} sisters.")

# print(r"Hello, am {r} Nice to meet you {sh}.")

if ag:
    print("Shobha is {ag} years old.")
else: print("Not sure of the age")


# coding=utf-8
# Copyleft 2020 project COL.

import argparse
import copy
from multiprocessing import Queue, Process
import os
import queue
import sys
import time

import h5py
import torch
import tqdm
from spacy.lang.en import English
import numpy as np

# sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from vokenization import load_model_and_tokenizer, Vokenizer

# Handle the GPU issue in multi-processing.
from multiprocessing import set_start_method
try:
    set_start_method('spawn')
except RuntimeError:
    pass

def processer(args, input_queue, output_queue):
    print(f"Setup workers on gpu {args.gpus}")
    img_sets = sorted([img_set.strip() for img_set in args.image_sets.split(',')])

    print("Build models and tokenizer")
    # We will assign the GPU to model latter, thus load to cpu first!
    model, tokenizer = load_model_and_tokenizer(args.load, cpu=True)
    keys_dir = args.load + '/keys'  # Save the keys with the model dict

    print("Build Retriever from %s with image sets" % keys_dir, img_sets)
    sentence_level = ('sent' in args.load)
    vokenizer = Vokenizer(model, tokenizer, keys_dir,
                          img_sets=img_sets, max_img_num=args.max_img_num,
                          gpus=args.gpus, sent_level=sentence_level)
    print(f"GPU: {args.gpus}, build vokenizer with {vokenizer.img_num} images.")

    # Before vokenization, save the image ids
    dset_name = os.path.split(args.corpus)[-1]
    modifier = f".{vokenizer.img_num}" if vokenizer.img_num != 50000 else ""
    vokens_img_ids_path = os.path.join(
        args.output,
        f"{dset_name}.{'_'.join(img_sets)}{modifier}.ids"
    )
    if args.gpus[0] == 0:
        if os.path.exists(vokens_img_ids_path):
            # If the img_ids file exists, assert that they are the same.
            saved_img_ids = open(vokens_img_ids_path).readlines()
            img_ids = vokenizer.img_ids
            assert len(saved_img_ids) == len(img_ids)
            for saved_img_id, img_id in zip(saved_img_ids, img_ids):
                assert saved_img_id.strip() == img_id
        else:
            vokenizer.dump_img_ids(vokens_img_ids_path)

    while True:
        page_id, sents = input_queue.get()
        # Print the first few sents for debugging
        if args.gpus[0] == 0:
            if page_id < 12 and sents is not None:
                print('page_id:', page_id)
                print('batch_size:', len(sents))
                print('ids of sent[0]:', sents[0])
                print('tokens of sent[0]:', tokenizer.convert_ids_to_tokens(sents[0]))
                print()
        # print(f"Processer {args.gpus}: Get Page Id {page_id}")
        # if sents is not None:
        #     output_str = ''
        #     results = vokenizer.vokenize_ids(sents)
        #     idxs = results[1]
        #     for j, idx in enumerate(idxs):
        #         assert len(idx[1:-1]) == len(sents[j])
        #         dump_idx = map(lambda x: str(x.item()), idx[1:-1])
        #         output_str += ' '.join(dump_idx) + '\n'

        #     output_queue.put((page_id, output_str))
        # else:
        #     break

        # Change by Jokie : cached the image path rather than the image_local_id
        topks = [5]
        arg_topk = None if max(topks) == 1 else max(topks)
        if sents is not None:
            output_str = ''
            results = vokenizer.vokenize_ids(sents, attention_mask=None, topk=arg_topk)
            paths_list = results[3]

            if sentence_level:
                paths_list = [x[0] for x in paths_list]     # Only eval the first vokens.

            # print('after path:',paths_list)
            if arg_topk is None:
                paths_list = [[[img_id] for img_id in sent] for sent in paths_list]

            for paths in paths_list:                # for each sent in batch
                top_k_line = '\t'.join([os.path.split(kth_path)[-1] for kth_path in paths])
                output_str += top_k_line + '\n'

            output_queue.put((page_id, output_str))
        else:
            break

def reducer(output_fname, output_queue, total_tokens):
    next_page_id = 0
    heap = queue.PriorityQueue()
    output = open(output_fname, 'a')
    cache = ""
    start_time = None
    processed_tokens = 0

    while True:
        page_id, result = output_queue.get()
        if start_time is None:      # The clock starts to tick when receiving the first package.
            start_time = time.time()
        # print("Reducer: Get Page Id %d" % page_id)
        if result is not None:
            # Put it into the heap
            heap.put((page_id, result))

            # Check the could-be-dumped data in the queue
            while heap.qsize() > 0:
                smallest_page_id, result = heap.get()
                if smallest_page_id == next_page_id:
                    # which means that this page is the next page, thus dump it
                    # print("Reducer: Commit Page Id %d" % next_page_id)
                    processed_tokens += len(result.split(' '))
                    cache += result
                    next_page_id += 1
                else:
                    heap.put((smallest_page_id, result))
                    break
            # print("Reducer: Length of Cache Now", len(cache))
            if len(cache) > 1000000:
                # Dump for every 1000000 characters to reduce IO calls
                output.write(cache)
                output.flush()
                cache = ''
                used_time = int(time.time() - start_time)
                print("Process %d tokens, %d to go, with speed %0.2f tokens/second,"
                      "finished in %0.2f hours" % (
                    processed_tokens,
                    total_tokens - processed_tokens,
                    processed_tokens / used_time,
                    (total_tokens - processed_tokens) / (processed_tokens / used_time) / 3600
                ))
        else:
            if len(cache) > 0:
                output.write(cache)
                output.flush()
                cache = ''
            break

    output.close()

def setup_mp(args, tokens, sent_ranges, vokens_path):
    QUEUE_SIZE = 10000
    input_queue = Queue(maxsize=QUEUE_SIZE)
    output_queue = Queue(maxsize=QUEUE_SIZE)

    workers = []
    num_gpu = torch.cuda.device_count()
    for worker_id in range(args.num_workers):
        gpu_id = worker_id % num_gpu
        curr_args = copy.copy(args)
        curr_args.gpus = (gpu_id,)
        worker = Process(target=processer,
                         args=(curr_args, input_queue, output_queue))
        worker.daemon = True
        worker.start()
        workers.append(worker)

    total_tokens = len(tokens) - sent_ranges[0][0] if len(sent_ranges) > 0 else 0
    reduce = Process(target=reducer,
                     args=(vokens_path, output_queue, total_tokens))
    reduce.start()

    for i, start_id in enumerate(range(0, len(sent_ranges), args.batch_size)):
        sents = []
        for left, right in sent_ranges[start_id: start_id + args.batch_size]:
            sents.append(tokens[left: right])
        input_queue.put((i, sents))

    # Notifying workers the end of input
    for _ in workers:
        input_queue.put((-1, None))

    # wait for workers to terminate
    for w in workers:
        w.join()

    # Notify the reducer the end of output
    output_queue.put((-1, None))

    # wait for reducer to terminate
    reduce.join()

def segment_sent(
        tokens,
        tokenizer,
        tokens_line_info_path,
        tokens_sent_info_path
    ):
    """
    Single-processed segmentation of sentences. We might need to parallel this as well.
    """
    with open(tokens_line_info_path) as f:
        line_starts = list(map(int, f.readlines()))

    nlp = English()
    sentencizer = nlp.create_pipe("sentencizer")
    nlp.add_pipe(sentencizer)

    sent_starts = [0]
    now = 0
    print("Now, split lines into sentences with Spacy:")
    for i in tqdm.tqdm(range(len(line_starts) - 1)):
        start_token_idx = line_starts[i]
        end_token_idx = line_starts[i + 1]
        line_tokens = tokens[start_token_idx: end_token_idx]
        line = ' '.join(tokenizer.convert_ids_to_tokens(line_tokens))
        line = line.replace("[UNK]", "UNK")

        doc = nlp(line)
        sents_len = 0
        sents = []
        for sent in doc.sents:
            if i < 2:
                print(sent)
                print('---------------')
            sent = str(sent)
            sents.append(sent)
            words = sent.split(' ')
            sent_len = len(words)
            now += sent_len
            sent_starts.append(now)
            sents_len += sent_len

        if sents_len != len(line_tokens):
            print('sents_len:',sents_len)
            print('sents:',sents)
            print('len(line_tokens):',len(line_tokens))
            print('line:',line)
            assert False
        assert sent_starts[-1] == end_token_idx

    with open(tokens_sent_info_path, 'w') as f:
        for sent_start in sent_starts:
            f.write(str(sent_start) + "\n")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    # Text
    parser.add_argument('--corpus', type=str, default='/ssd-playpen/data/wiki103/wiki.train.raw')
    # Models
    parser.add_argument('--load', type=str,
                        default='/ssd-playpen/home/hTan/CoL/CoX/snap/pretrain/coco_hinge05_dim64_resxt101_robertal4',
                        help='The directory saved the model (containing'
                             'BEST.pth.model).')
    parser.add_argument('--output', type=str, default=None,
                        help='The directory to save the extracted feature keys.'
                             '"None" would save in the "load" dir')
    parser.add_argument('--tokenizer-name', type=str, default='roberta-base')
    # Vision: Define the vokens set
    parser.add_argument('--image-sets', type=str, default='vg_nococo',
                        help='The splits of images to be extracted')
    parser.add_argument('--max-img-num', type=int, default=-1,
                        help='number of images used. -1 means all images.')
    # Speed Up Options:
    parser.add_argument('--num-workers', type=int, default=-1,
                        help='-1 will use all GPUs.')
    parser.add_argument('--batch-size', type=int, default=16,
                        help='The # of sentences in a batch.')
    args = parser.parse_args()

    if args.num_workers == -1:
        args.num_workers = torch.cuda.device_count()

    if args.output is None:
        args.output = os.path.join(args.load, 'vokens')
    os.makedirs(args.output, exist_ok=True)

    dset_name = os.path.split(args.corpus)[-1]
    img_sets = sorted([img_set.strip() for img_set in args.image_sets.split(',')])
    print()
    print("Main Th"
          "read: Build a virtual vokenizer to check the number of images.")
    keys_dir = args.load + '/keys'  # Save the keys with the model dict
    virtual_vokenizer = Vokenizer(
        None, None, keys_dir,
        img_sets=img_sets, max_img_num=args.max_img_num,
        gpus=(-1,), sent_level=('sent' in args.load))
    modifier = f".{virtual_vokenizer.img_num}" if virtual_vokenizer.img_num != 50000 else ""
    vokens_path = os.path.join(
        args.output,
        f"{dset_name}.{'_'.join(img_sets)}{modifier}"
    )
    tokens_hdf5_path = f'{args.corpus}.{args.tokenizer_name}.hdf5'
    # tokens_sent_info_path = f'{args.corpus}.{args.tokenizer_name}.sent'
    # Chenged by Jokie: use the whole line to retrieval the iamge
    tokens_sent_info_path = f'{args.corpus}.{args.tokenizer_name}.line'

    # "Load" tokens from hdf5
    tokens_hdf5 = h5py.File(tokens_hdf5_path, 'r')
    tokens = tokens_hdf5['tokens']

    # Calibrate the start line if the vokens have been proceeded.
    if not os.path.exists(tokens_sent_info_path):
        #comment by Jokie
        # tokens_line_info_path = f'{args.corpus}.{args.tokenizer_name}.line'
        # model, tokenizer = load_model_and_tokenizer(args.load, cpu=True)
        # segment_sent(
        #     tokens,
        #     tokenizer,
        #     tokens_line_info_path,
        #     tokens_sent_info_path
        # )
        pass

    # Load sent info and find the start sentence
    with open(tokens_sent_info_path) as f:
        sent_starts = list(map(int, f.readlines()))

    # Skip the sentences which have been extracted.
    extracted_tokens = 0
    if os.path.isfile(vokens_path):
        with open(vokens_path, 'r') as g:
            for g_line in tqdm.tqdm(g):
                extracted_tokens += len(g_line.strip().split(' '))
    try:
        start_sent_idx = sent_starts.index(extracted_tokens)
    except ValueError as e:
        print("The extracted tokens does not match a starting sentence.")
        print(e)

    # Start to vokenize
    print("Main Thread: Dump visual tokens to %s" % vokens_path)
    print("Main Thread: Start vokenization from the %d'th token" % sent_starts[start_sent_idx])

    sent_ranges = []
    for i in range(start_sent_idx, len(sent_starts) - 1):
        left_token_idx = sent_starts[i]
        right_token_idx = sent_starts[i + 1]
        sent_ranges.append((left_token_idx, right_token_idx))

    setup_mp(args, tokens, sent_ranges, vokens_path)

    # # save into hdf5 file
    # if os.path.exists(vokens_path + '.hdf5'):
    #     print("The hdf5 file %s already exists. So the hdf5 file is not converted."
    #           % (vokens_path + '.hdf5'))
    # else:
    #     with open(args.corpus + '.' + args.tokenizer_name + ".sent") as f:
    #         for i, line in enumerate(f):
    #             pass
    #         num_tokens = int(line)
    #         num_sents = i

    #     h5_file = h5py.File(vokens_path + '.hdf5', 'w')
    #     # dset = h5_file.create_dataset("vokens", (num_tokens,), dtype='int32')
    #     dset = h5_file.create_dataset("vokens", (num_tokens,), dtype=np.str)

    #     dump_interval = 100000
    #     dump_iter = 0
    #     lines = 0

    #     with open(vokens_path) as f:
    #         tokens = []
    #         for line in tqdm.tqdm(f, total=num_sents):
    #             for token in map(int, line.split('\t')):
    #                 tokens.append(token)
    #             if len(tokens) >= dump_interval:
    #                 dset[dump_iter: dump_iter + len(tokens)] = tokens
    #                 dump_iter += len(tokens)
    #                 tokens = []
    #             lines += 1
    #         dset[dump_iter: dump_iter + len(tokens)] = tokens
    #         dump_iter += len(tokens)
    #         assert num_tokens == dump_iter
    #         print(lines, num_sents)
    #         assert lines == num_sents
    #     h5_file.close()


"""
Contains PersonalDetails class that provides help with
interacting with the Edit Account page.
"""

class PersonalDetails:
    """
    Use to edit user's personal info
    on the Edit Account page.
    """

    def __init__(self,
                 firstname=None,
                 lastname=None,
                 email=None,
                 telephone=None):
        self.firstname = firstname
        self.lastname = lastname
        self.email = email
        self.telephone = telephone

    def __repr__(self):
        return "{} {} {} {}".format(self.firstname,
                                    self.lastname,
                                    self.email,
                                    self.telephone)

    def __eq__(self, other):
        return self.firstname == other.firstname \
               and self.lastname == other.lastname \
               and self.email == other.email \
               and self.telephone == other.telephone


#! /usr/bin/python3
# -*- coding: UTF-8 -*-

import time, random, sqlite3

def sqliteDBAPI (db_file, sqlite_cmdline, args = None):

    flag = True;
    while flag == True:
        flag = False;
        if args == None:
            db_connection = sqlite3.connect (db_file);
            try:
                ret_list = [];
                #wair for unlock
                for item in db_connection.execute (sqlite_cmdline):
                    ret_list.append (item);
                db_connection.commit ();
            except BaseException as e:
                db_connection.rollback ();
                ret_list = None;
                if 'database is locked' in str (e):
                    flag = True;
                    time.sleep (random.randint (3, 5));
                else:
                    print ('Error Commend Line Is :');
                    print (sqlite_cmdline);
                    print ('Error Resean Is :');
                    print (e);
            finally:
                db_connection.close ();
                return ret_list;
        else:
            db_connection = sqlite3.connect (db_file);
            try:
                #wair for unlock
                db_connection.execute (sqlite_cmdline, args);
                db_connection.commit ();
            except BaseException as e:
                db_connection.rollback ();
                if 'database is locked' in str (e):
                    flag = True;
                    time.sleep (random.randint (3, 5));
                else:
                    print ('Error Resean Is :');
                    print (e);
                    if 'probably unsupported type' in str (e):
                        print ('args item is :');
                        number = 0;
                        for item in args:
                            print (number, ':', type (item), ':', item);
                            number += 1;
                        print ('args item over')
            finally:
                db_connection.close ();

def sqliteDBMANYAPI (db_file, sqlite_cmdline, args):

    flag = True;
    #wair for unlock
    while flag == True:
        flag = False;
        db_connection = sqlite3.connect (db_file);
        try:
            db_connection.executemany (sqlite_cmdline, args);
            db_connection.commit ();
        except BaseException as e:
            db_connection.rollback ();
            if 'database is locked' in str (e):
                flag = True;
                time.sleep (random.randint (3, 5));
            else:
                print ('Error Resean Is :');
                print (e);
        finally:
            db_connection.close ();

def main ():

    db_file = 'testxp1024.db';
    sqlite_cmdline = \
        '''CREATE TABLE IF NOT EXISTS PAGE_DB
        (PAGEINDEX INT PRIMARY KEY NOT NULL,
        URL TEXT,
        NAME TEXT,
        POST TEXT,
        DONEFLAG INT
        );''';

    sqliteDBAPI (db_file, sqlite_cmdline);
    sqlite_cmdline = \
        '''CREATE TABLE IF NOT EXISTS POST_DB
        (URL TEXT PRIMARY KEY NOT NULL,
        NAME TEXT,
        MOVIE TEXT,
        DONEFLAG INT
        );''';

    sqliteDBAPI (db_file, sqlite_cmdline);
    sqlite_cmdline = \
        '''CREATE TABLE IF NOT EXISTS MOVIE_DB
        (URL TEXT PRIMARY KEY NOT NULL,
        DESCRIPTION TEXT,
        PICTURE TEXT,
        DONEFLAG INT
        );''';

    sqliteDBAPI (db_file, sqlite_cmdline);

    return;

if __name__ == '__main__':

    main ();


from django import forms
from portal.models import Absence
import datetime as dt

class SubmitForm(forms.ModelForm):

    def clean(self):
        """Validate from_date is lesser than to_date"""
        cleaned_data = super().clean()
        from_date = cleaned_data.get('from_date')
        to_date = cleaned_data.get('to_date')

        days_range = (to_date.date() - from_date.date()).days

        if (days_range < 0):
            raise forms.ValidationError("From date must be lesser than To date!")

    class Meta:
        model = Absence
        fields = ("from_date", "to_date", "type_miss", "comment")
        widgets = {
            'from_date': forms.TextInput(attrs={'class': 'form-control date-picker-js', 'required': True}),
            'to_date': forms.TextInput(attrs={'class': 'form-control date-picker-js', 'required': True}),
            'type_miss': forms.Select(attrs={'class': 'form-control', 'required': True}),
            'comment': forms.TextInput(attrs={'class': 'form-control', 'required': False})
        }


from Node import *
import check

## A class demonstrating a linked list implementation of a
## sequence resembling Python lists.
class LinkedList:
    '''
    Field: _head referring to the first node of a linked list
    '''

    ## __init__: None -> LinkedList
    ## Produces a newly constructed empty linked list.
    def __init__(self):
        self._head = None

    ## __repr__: LinkedList -> string
    ## Consumes self and produces a string consisting
    ## of the items of self between square brackets
    ## separated by commas.
    def __repr__(self):
        s = "["
        refNode = self._head
        if refNode is not None:
            s = s + str(refNode.item)
            refNode = refNode.next
        while refNode is not None:
            s = s + "," + str(refNode.item) # inefficient
            refNode = refNode.next              # string
        return s + "]"                  # concatenation

    ## __get_item__: LinkedList int -> any
    ## Consumes self and a valid index.
    ## Produces the item in self at that index.
    def __getitem__(self, index):
        refNode = self._head
        for i in range(index): # could reuse later if
            refNode = refNode.next     # put into helper method
        return refNode.item

    ## __len__: LinkedList -> int
    ## Consumes self and produces the number of items
    ## in self.
    def __len__(self): # alternative: maintain a length field
        count = 0
        refNode = self._head
        while refNode is not None:
            count = count + 1
            refNode = refNode.next
        return count

    ## append: LinkedList any -> None
    ## Consumes self and an item and adds the item
    ## to the end of self.
    def append(self, x):
        if self._head is None:
            self._head = Node(x)
        else:
            refNode = self._head
            while refNode.next is not None:
                refNode = refNode.next
            refNode.next = Node(x)

    ## extend: LinkedList LinkedList -> None
    ## Consumes self and other and extends self
    ## by appending the items of other to the end
    ## of self in order.
    def extend(self, other):
        if self._head is None:
            self._head = other._head
        else:
            refNode = self._head
            while refNode.next is not None:
                refNode = refNode.next
            refNode.next = other._head
        ## Note: self and other share memory
        ##       unlike with Python lists

    ## insert: LinkedList int any -> None
    ## Consumes self, a valid index and item and
    ## adds item to this index in the list.
    def insert(self, index, item):
        if index == 0:
            self._head = Node(item,self._head)
        else:
            refNode = self._head
            for i in range(index-1):
                refNode = refNode.next
            refNode.next = Node(item,refNode.next)

    ## pop: LinkedList int -> None
    ## Consumes self and a valid index and removes
    ## the item in the list at this index.
    def pop(self, index):
        if index == 0:
            self._head = self._head.next
        else:
            refNode = self._head
            for i in range(index-1):
                refNode = refNode.next
            refNode.next = refNode.next.next

    ## remove: LinkedList any -> None
    ## Consumes self and an item and removes
    ## the first equal item in the list (if any).
    def remove(self, item):
        refNode = self._head
        prevRef = None
        while refNode is not None and refNode.item != item:
            prevRef = refNode
            refNode = refNode.next
        if refNode is not None:
            if prevRef is None:
                self._head = self._head.next
            else:
                prevRef.next = refNode.next

    ## reverse: LinkedList -> None
    ## Consumes self and reverses it.
    def reverse(self):
        if self._head is not None:
            temp = self._head.item
            self._head = self._head.next
            self.reverse()
            self.append(temp)

    # The three methods you must write are below

    ## count: LinkedList any -> int[>=0]
    ##
    ## Produces the number of ocrefNoderences of x in self.
    ## (i.e. the number of items y in self for which x==y)
    ##
    ## Example:
    ##   If LL is a linked list and print LL displays ['hi',55,3,55,-4],
    ##   then LL.count(55) => 2
    def count(self, x):
        refNode = self._head
        num = 0
        while refNode is not None:
            if refNode.item == x:
                num = num + 1
            refNode = refNode.next
        return num

    ## __mul__: LinkedList int[>=1] -> LinkedList
    ##
    ## Consumes k and self where all items in self are integers.
    ## Produces a new linked list containing all the integers in self
    ## repeated k times. The produced list and self do not share
    ## any memory (nodes).
    ##
    ## Example:
    ##   If LL is a linked list and print LL displays [17,8,9,22,5],
    ##   then print LL*3 displays [17,8,9,22,5,17,8,9,22,5,17,8,9,22,5]
    def __mul__(self, k):
        result = LinkedList()

        if self._head is not None:
            # set head of result
            result._head = Node(self._head.item)
            resulttail = result._head

            for i in range(k):
                selfrefNode = self._head
                if i == 0:   # skip head first time
                    selfrefNode = selfrefNode.next
                while selfrefNode is not None:
                    resulttail.next = Node(selfrefNode.item)
                    selfrefNode = selfrefNode.next
                    resulttail = resulttail.next

        return result
        # alternative approach: do all the inserting at the front of
        # the list and then reverse the list in linear time (i.e. not
        # using the given implementation of reverse).

    ## filter_out: LinkedList (any -> bool) -> None
    ##
    ## Consumes self and f where f(x) produces True or False
    ## for all items x in self. Mutates self by removing all items x
    ## in self such that f(x) is True.
    ##
    ## Example:
    ##   If LL is a linked list and print LL displays [1,2,3,4,5],
    ##   and then after LL.filter_out(lambda x : (x > 1 and x < 4)) is called,
    ##   print LL displays [1,4,5]
    def filter_out(self, f):
         # filter out all items at front of list
         while self._head is not None and f(self._head.item):
             self._head = self._head.next

         # filter out all remaining items
         prevRef = None
         refNode = self._head
         while refNode is not None:
             if f(refNode.item):
                 prevRef.next = refNode.next
                 refNode = prevRef
             prevRef = refNode
             refNode = refNode.next

# Sampling of tests
LL = LinkedList()
LL.append('hi')
LL.append(55)
LL.append(3)
LL.append(55)
LL.append(-4)
check.expect("count Sample Test 1",LL.count(55),2)
check.expect("count Sample Test 2",LL.count('hi'),1)
check.expect("count Sample Test 3",LL.count(-3),0)

LL = LinkedList()
LL.append(1)
LL.append(2)
LL.append(3)
check.expect("__mul__ Sample Test 1",str(LinkedList()*7),"[]")
check.expect("__mul__ Sample Test 2",str(LL*1),"[1,2,3]")
check.expect("__mul__ Sample Test 3",str(LL*3),"[1,2,3,1,2,3,1,2,3]")

LL1 = LinkedList()
LL1.append(1)
LL1.append(2)
LL1.append(3)
LL2 = LL1 * 3
LL2.append(1)
LL2.filter_out(lambda x : x == 1)
check.expect("filter_out Sample Test",str(LL2),"[2,3,2,3,2,3]")


import sys
import os.path
import time

# TODO: it is better to return (file_name, x)
# as the result of the argument parsing
def checkArgument():
    if len(sys.argv) != 3:
        print ("Incorrect number of arguments")
        printUsage()

    filename = sys.argv[1]
    xstring = sys.argv[2]
    if not os.path.isfile(filename):
        print ('File %s cannot be opened' % filename)
        printUsage()

    try:
        x = int(xstring)
        return (filename, x)
    except ValueError:
        print ("Third argument %s not integer" % xstring)
        printUsage()

def printUsage():
    print ("Usage: %s filename integer" % sys.argv[0])
    sys.exit()

"""
Reads the file into a list of ranges.
"""
def ReadFile(file_name):
    file = open(file_name, 'r')
    range_list = []
    for line in file:
        (r, l) = line.split()
        range = (int(r), int(l))
        range_list.append(range)
    return range_list

"""
Creates a map of RangeBoundries using a list of ranges.
"""
def CreateBoundaryMap(range_list):
    boundry_map = {}
    for range in range_list:
        if range[0] not in boundry_map:
            boundry_map[range[0]] = RangeBoundry()
        boundry_map[range[0]].insert_list_.append(range)
        if range[1] not in boundry_map:
            boundry_map[range[1]] = RangeBoundry()

        if range[1] + 1 not in boundry_map:
            boundry_map[range[1] + 1] = RangeBoundry()
        boundry_map[range[1] + 1].remove_list_.append(range)
    return boundry_map

"""
Creates a list of numbers in order from the lower and upper bounds of a
list of ranges.
"""
def CreateNumList(range_list):
    num_list = []
    for range in range_list:
        if range[0] not in num_list:
            num_list.append(range[0])
        if range[1] not in num_list:
            num_list.append(range[1])
        if range[1] + 1 not in num_list:
            num_list.append(range[1] + 1)
    num_list.sort()
    return num_list

"""
Creates a list of NewRanges containing the original ranges.
"""
def CreateNewRanges(num_list, boundry_map):
    new_ranges = []
    for x in range(len(num_list) - 1):
        new_ranges.append(NewRange(num_list[x], num_list[x+1] - 1))
    current_set = []
    for entry in new_ranges:
        current_set.extend(boundry_map[entry.low_].insert_list_)
        for ranges in boundry_map[entry.low_].remove_list_:
            current_set.remove(ranges)
        entry.rangelist_.extend(current_set)

    return new_ranges

"""
The NewRange class contains and upper and lower bounds, as well as a list of
the original ranges that overlap it.
"""
class NewRange:
    def __init__(self, lower, upper):
        self.low_ = lower
        self.high_ = upper
        self.rangelist_ = []

"""
The RangeBoundry class includes two lists consisting of ranges that need to be
added or removed during InsertSort at a certain number
"""
class RangeBoundry:

    def __init__(self):
        self.insert_list_ = []
        self.remove_list_ = []

class RangeFinder:
    def __init__(self, range_list):
        boundary_map = CreateBoundaryMap(range_list)
        num_list = CreateNumList(range_list)
        self.new_ranges = CreateNewRanges(num_list, boundary_map)

    def LookUp(self, x):
        return self.LookUpHelper(0, len(self.new_ranges), x)

    def LookUpHelper(self, start, end, x):
        if not self.new_ranges:
            return []
        halfway = int((start + end)/2)
        (r, l) = (self.new_ranges[halfway].low_, self.new_ranges[halfway].high_)
        if x >= r and x <= l:
            return self.new_ranges[halfway].rangelist_
        if end - start == 1:
            return []
        if x < r:
            return self.LookUpHelper(start, halfway, x)
        if x > l:
            return self.LookUpHelper(halfway, end, x)

"""
LookUp finds if a given number is within the ranges in a given list, and if
so, what original ranges are associated with it.
"""
def LookUp(list, start, end, x):
    if not list:
        return []
    halfway = int((start + end)/2)
    (r, l) = (list[halfway].low_, list[halfway].high_)
    if x >= r and x <= l:
        return list[halfway].rangelist_
    if end - start == 1:
        return []
    if x < r:
        return LookUp(list, start, halfway, x)
    if x > l:
        return LookUp(list, halfway, end, x)

if __name__ == '__main__':
    (file_name, x) = checkArgument()
    systemtime = time.clock()
    range_list = ReadFile(file_name)
    boundary_map = CreateBoundaryMap(range_list)
    num_list = CreateNumList(range_list)
    new_ranges = CreateNewRanges(num_list, boundary_map)

    finder = RangeFinder(range_list)
    # for entry in listofranges:
        # print (entry.rangelist)
    #ranges = LookUp(new_ranges, 0, len(new_ranges), x)
    ranges = finder.LookUp(x)
    if ranges:
        print ('\n', ranges)
    else:
        print ('\n Not found')

    # print (time.clock() - systemtime)


# -*- mode:python; coding:utf-8 -*-

# Copyright (c) 2020 IBM Corp. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for trestle load_distributed module."""

import shutil
from pathlib import Path

import dictdiffer

from tests import test_utils

from trestle.oscal.catalog import Catalog, ResponsibleParty, Role
from trestle.utils import fs
from trestle.utils.load_distributed import _load_dict, _load_list, load_distributed

def test_load_list(testdata_dir, tmp_trestle_dir):
    """Test loading a list recursively."""
    # prepare trestle project dir with the file
    test_utils.ensure_trestle_config_dir(tmp_trestle_dir)

    test_data_source = testdata_dir / 'split_merge/step4_split_groups_array/catalogs'

    catalogs_dir = Path('catalogs/')
    mycatalog_dir = catalogs_dir / 'mycatalog'
    catalog_dir = mycatalog_dir / 'catalog'

    # Copy files from test/data/split_merge/step4
    shutil.rmtree(catalogs_dir)
    shutil.copytree(test_data_source, catalogs_dir)

    actual_model_type, actual_model_alias, actual_roles = _load_list(catalog_dir / 'metadata' / 'roles')

    expected_roles = [
        Role.oscal_read(catalog_dir / 'metadata/roles/00000__role.json'),
        Role.oscal_read(catalog_dir / 'metadata/roles/00001__role.json')
    ]

    expected_model_type, _ = fs.get_stripped_contextual_model((catalog_dir / 'metadata/roles').absolute())

    assert actual_model_type.__signature__ == expected_model_type.__signature__
    assert actual_model_alias == 'catalog.metadata.roles'
    assert test_utils.list_unordered_equal(actual_roles, expected_roles)

def test_load_list_group(testdata_dir, tmp_trestle_dir):
    """Test more complicated list loading."""
    # prepare trestle project dir with the file
    test_utils.ensure_trestle_config_dir(tmp_trestle_dir)

    test_data_source = testdata_dir / 'split_merge/step4_split_groups_array/catalogs'

    catalogs_dir = Path('catalogs/')
    mycatalog_dir = catalogs_dir / 'mycatalog'
    catalog_dir = mycatalog_dir / 'catalog'

    # Copy files from test/data/split_merge/step4
    shutil.rmtree(catalogs_dir)
    shutil.copytree(test_data_source, catalogs_dir)

    actual_model_type, actual_model_alias, actual_groups = _load_list(catalog_dir / 'groups')

    # load_list is expected to return a list of array, instead of an instance of Groups class
    expected_groups = (actual_model_type.oscal_read(testdata_dir / 'split_merge/load_distributed/groups.json')).__root__

    assert actual_groups == expected_groups

def test_load_dict(testdata_dir, tmp_trestle_dir):
    """Test loading of distributed dict."""
    # prepare trestle project dir with the file
    test_utils.ensure_trestle_config_dir(tmp_trestle_dir)

    test_data_source = testdata_dir / 'split_merge/step4_split_groups_array/catalogs'

    catalogs_dir = Path('catalogs/')
    mycatalog_dir = catalogs_dir / 'mycatalog'
    catalog_dir = mycatalog_dir / 'catalog'

    # Copy files from test/data/split_merge/step4
    shutil.rmtree(catalogs_dir)
    shutil.copytree(test_data_source, catalogs_dir)

    actual_model_type, actual_model_alias, actual_model_instance = _load_dict(
        catalog_dir / 'metadata/responsible-parties')

    expexted_model_instance = {
        'contact': ResponsibleParty.oscal_read(
            catalog_dir / 'metadata/responsible-parties/contact__responsible-party.json'
        ),
        'creator': ResponsibleParty.oscal_read(
            catalog_dir / 'metadata/responsible-parties/creator__responsible-party.json'
        )
    }
    assert len(list(dictdiffer.diff(actual_model_instance, expexted_model_instance))) == 0
    assert actual_model_alias == 'catalog.metadata.responsible-parties'

    expected_model_type, _ = fs.get_contextual_model_type((catalog_dir / 'metadata/responsible-parties/').absolute())
    assert actual_model_type.__fields__['__root__'].outer_type_ == expected_model_type

def test_load_distributed(testdata_dir, tmp_trestle_dir):
    """Test massive distributed load, that includes recusive load, list and dict."""
    # prepare trestle project dir with the file
    test_utils.ensure_trestle_config_dir(tmp_trestle_dir)

    test_data_source = testdata_dir / 'split_merge/step4_split_groups_array/catalogs'

    catalogs_dir = Path('catalogs/')
    mycatalog_dir = catalogs_dir / 'mycatalog'
    catalog_file = mycatalog_dir / 'catalog.json'

    # Copy files from test/data/split_merge/step4
    shutil.rmtree(catalogs_dir)
    shutil.copytree(test_data_source, catalogs_dir)

    actual_model_type, actual_model_alias, actual_model_instance = load_distributed(catalog_file)

    expected_model_type, _ = fs.get_contextual_model_type(catalog_file.absolute())

    expected_model_instance = Catalog.oscal_read(testdata_dir / 'split_merge/load_distributed/catalog.json')

    assert actual_model_type == expected_model_type
    assert actual_model_alias == 'catalog'
    assert len(list(dictdiffer.diff(expected_model_instance, actual_model_instance))) == 0


"""
Mask R-CNN
Train on the toy Balloon dataset and implement color splash effect.

Copyright (c) 2018 Matterport, Inc.
Licensed under the MIT License (see LICENSE for details)
Written by Waleed Abdulla

------------------------------------------------------------

Usage: import the module (see Jupyter notebooks for examples), or run from
       the command line as such:

    # Train a new model starting from pre-trained COCO weights
    python3 balloon.py train --dataset=/path/to/balloon/dataset --weights=coco

    # Resume training a model that you had trained earlier
    python3 balloon.py train --dataset=/path/to/balloon/dataset --weights=last

    # Train a new model starting from ImageNet weights
    python3 balloon.py train --dataset=/path/to/balloon/dataset --weights=imagenet

    # Apply color splash to an image
    python3 balloon.py splash --weights=/path/to/weights/file.h5 --image=<URL or path to file>

    # Apply color splash to video using the last weights you trained
    python3 balloon.py splash --weights=last --video=<URL or path to file>
"""

#import os
#import sys
#import json
#import datetime
#import numpy as np
#import skimage.draw
#
#import matplotlib # bai
#import matplotlib.pyplot as plt
#import matplotlib.patches as patches
#
## Root directory of the project
#ROOT_DIR = os.path.abspath("../../")
#
## Import Mask RCNN
#sys.path.append(ROOT_DIR)  # To find local version of the library
#from mrcnn.config import Config
#from mrcnn import model as modellib, utils
#
#from mrcnn import visualize # bai
#from mrcnn.visualize import display_images # bai
#
#
## Path to trained weights file
#COCO_WEIGHTS_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")
#
## Directory to save logs and model checkpoints, if not provided
## through the command line argument --logs
#DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs")

import os
import sys
import json
import datetime
import numpy as np
import skimage.draw
from imgaug import augmenters as iaa

import matplotlib#GAI
import matplotlib.pyplot as plt#GAI
import matplotlib.patches as patches#GAI

# Root directory of the project
ROOT_DIR = os.path.abspath("../../")

# Import Mask RCNN
sys.path.append(ROOT_DIR)  # To find local version of the library
from my_mrcnn.config import Config#GAI
from my_mrcnn import model as modellib, utils#GAI

from my_mrcnn import visualize #GAI
from my_mrcnn.visualize import display_images#GAI

# Path to trained weights file
COCO_WEIGHTS_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")

# Directory to save logs and model checkpoints, if not provided
# through the command line argument --logs
DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs")
############################################################
#  Configurations
############################################################

class sicConfig(Config):#GAI
    """Configuration for training on the toy  dataset.
    Derives from the base Config class and overrides some values.
    """
    # Give the configuration a recognizable name
    NAME = "sic"#GAI

    # We use a GPU with 12GB memory, which can fit two images.
    # Adjust down if you use a smaller GPU.
    IMAGES_PER_GPU = 1

    # Number of classes (including background)
    NUM_CLASSES = 1 + 3  # Background + sic

    STEPS_PER_EPOCH = 100

    VALIDATION_STEPS = 50

#=========================================================
#train hyper-parameter

    #1
    BACKBONE = "resnet50"

    #2
    #Length of square anchor side in pixels
    RPN_ANCHOR_SCALES = (8, 16, 32, 64, 128)

    #3
    # Maximum number of ground truth instances to use in one image
    MAX_GT_INSTANCES = 500

    #4
    # Percent of positive ROIs used to train classifier/mask heads
    ROI_POSITIVE_RATIO = 0.66
##    ROI_POSITIVE_RATIO = 0.8
##    ROI_POSITIVE_RATIO = 1

#inference hyper-parameter

    #1
    # Max number of final detections
    DETECTION_MAX_INSTANCES = 500
#
#
#
    #2
    #Non-maximum suppression threshold for detection
    DETECTION_NMS_THRESHOLD = 0.5
##    DETECTION_NMS_THRESHOLD = 0.7
##    DETECTION_NMS_THRESHOLD = 0.9
#
    #3
    #Minimum probability value to accept a detected instance
    #ROIs below this threshold are skipped
    DETECTION_MIN_CONFIDENCE = 0
##    DETECTION_MIN_CONFIDENCE = 0.3
##    DETECTION_MIN_CONFIDENCE = 0.5

#====================================================================
#    LOSS_WEIGHTS = {
#        "rpn_class_loss": 1.,
#        "rpn_bbox_loss": 1.,
#        "mrcnn_class_loss": 1.,
#        "mrcnn_bbox_loss": 1.,
#        "mrcnn_mask_loss": 1.
#    }

#    #1
#    BACKBONE = "resnet50"
#
#    #2
#    #Length of square anchor side in pixels
#    RPN_ANCHOR_SCALES = (8, 16, 32, 64, 128)
#
#
#    #3
#    # Maximum number of ground truth instances to use in one image
#    MAX_GT_INSTANCES = 500
#
#
#    #4
#    # Percent of positive ROIs used to train classifier/mask heads
#    ROI_POSITIVE_RATIO = 0.66
#
#
#
#
#    #1
#    # Max number of final detections
#    DETECTION_MAX_INSTANCES = 500
#
#
#
#    #2
#    #Non-maximum suppression threshold for detection
#    DETECTION_NMS_THRESHOLD = 0.2
#
#    #3
#    #Minimum probability value to accept a detected instance
#    #ROIs below this threshold are skipped
#    DETECTION_MIN_CONFIDENCE = 0.5

############################################################
#  Dataset
############################################################

class sicDataset(utils.Dataset): # bai

    def load_sic(self, dataset_dir, subset): # bai
        """Load a subset of the sic dataset.
        dataset_dir: Root directory of the dataset.
        subset: Subset to load: train or val
        """
        # Add classes.
#        self.add_class("sic", 1, "BPD") # bai
#        self.add_class("sic", 2, "TSD") # bai
#        self.add_class("sic", 3, "TED") # bai
        self.add_class("sic", 1, "bpd") # bai
        self.add_class("sic", 2, "tsd") # bai
        self.add_class("sic", 3, "ted") # bai

        # Train or validation dataset?
        assert subset in ["train", "val", "test"]
        dataset_dir = os.path.join(dataset_dir, subset)

        # Load annotations
        # VGG Image Annotator (up to version 1.6) saves each image in the form:
        # { 'filename': '28503151_5b5b7ec140_b.jpg',
        #   'regions': {
        #       '0': {
        #           'region_attributes': {},
        #           'shape_attributes': {
        #               'all_points_x': [...],
        #               'all_points_y': [...],
        #               'name': 'polygon'}},
        #       ... more regions ...
        #   },
        #   'size': 100202
        # }
        # We mostly care about the x and y coordinates of each region
        # Note: In VIA 2.0, regions was changed from a dict to a list.
        annotations = json.load(open(os.path.join(dataset_dir, "via_region_data.json")))
        annotations = list(annotations.values())  # don't need the dict keys

        # The VIA tool saves images in the JSON even if they don't have any
        # annotations. Skip unannotated images.
        annotations = [a for a in annotations if a['regions']]

        # Add images
        for a in annotations:
            # Get the x, y coordinaets of points of the polygons that make up
            # the outline of each object instance. These are stores in the
            # shape_attributes (see json format above)
            # The if condition is needed to support VIA versions 1.x and 2.x.
            if type(a['regions']) is dict:
                polygons = [r['shape_attributes'] for r in a['regions'].values()]
                objects = [s['region_attributes'] for s in a['regions'].values()]  # bai
            else:
                polygons = [r['shape_attributes'] for r in a['regions']]
                objects = [s['region_attributes'] for s in a['regions']]  # bai

            class_ids = [int(n['sic']) for n in objects] # bai
            #class_ids = [int(n['defect']) for n in objects] # bai

            # load_mask() needs the image size to convert polygons to masks.
            # Unfortunately, VIA doesn't include it in JSON, so we must read
            # the image. This is only managable since the dataset is tiny.
            # print("multi_numbers=", multi_numbers)
            # num_ids = [n for n in multi_numbers['number'].values()]
            # for n in multi_numbers:

            image_path = os.path.join(dataset_dir, a['filename'])
            image = skimage.io.imread(image_path)
            height, width = image.shape[:2]

            self.add_image(
                "sic",
                image_id=a['filename'],  # use file name as a unique image id
                path=image_path,
                width=width, height=height,
                polygons=polygons,
                class_ids=class_ids)   # bai

    def load_mask(self, image_id):
        """Generate instance masks for an image.
       Returns:
        masks: A bool array of shape [height, width, instance count] with
            one mask per instance.
        class_ids: a 1D array of class IDs of the instance masks.
        """
        # If not a balloon dataset image, delegate to parent class.
        image_info = self.image_info[image_id]
        if image_info["source"] != "sic":
            return super(self.__class__, self).load_mask(image_id)
        class_ids = image_info['class_ids']  # bai
        # Convert polygons to a bitmap mask of shape
        # [height, width, instance_count]
        info = self.image_info[image_id]
        mask = np.zeros([info["height"], info["width"], len(info["polygons"])],
                        dtype=np.uint8)
        for i, p in enumerate(info["polygons"]):
            # Get indexes of pixels inside the polygon and set them to 1
            rr, cc = skimage.draw.polygon(p['all_points_y'], p['all_points_x'])
            mask[rr, cc, i] = 1

        # Return mask, and array of class IDs of each instance. Since we have
        # one class ID only, we return an array of 1s
        # class_ids=np.array([self.class_names.index(shapes[0])])
       # print("info['class_ids']=", info['class_ids'])
        class_ids = np.array(class_ids, dtype=np.int32)
        return mask, class_ids  # [mask.shape[-1]] #np.ones([mask.shape[-1]], dtype=np.int32)#class_ids.astype(np.int32)

    def image_reference(self, image_id):
        """Return the path of the image."""
        info = self.image_info[image_id]
        if info["source"] == "sic": # bai
            return info["path"]
        else:
            super(self.__class__, self).image_reference(image_id)

def train(model):
    """Train the model."""
    # Training dataset.
    dataset_train = sicDataset() # bai
    dataset_train.load_sic(args.dataset, "train") # bai
    dataset_train.prepare()

    # Validation dataset
    dataset_val = sicDataset() # bai
    dataset_val.load_sic(args.dataset, "val") # bai
    dataset_val.prepare()

    # *** This training schedule is an example. Update to your needs ***
    # Since we're using a very small dataset, and starting from
    # COCO trained weights, we don't need to train too long. Also,
    # no need to train all layers, just the heads should do it.

#==========
#     augmentation = iaa.SomeOf((0, None), [
#         iaa.Fliplr(1.0),
#         iaa.Flipud(1.0),
#         iaa.OneOf([iaa.Affine(rotate=90),
#                     iaa.Affine(rotate=180),
#                     iaa.Affine(rotate=270)]), #1
# #        iaa.Multiply((0.9, 1.1),per_channel=1.0), #2 (0.8,1.2)
# #        iaa.GammaContrast((0.8, 1.25)), #3 (0.75,1.3)
#         iaa.AdditiveGaussianNoise(scale=(0, 0.05*255)) #4
#     ], random_order=True)
#==========

    print("Training network heads")
    model.train(dataset_train, dataset_val,
                learning_rate=config.LEARNING_RATE,
                epochs=10, # 30,  bai
#                augmentation=augmentation, #here
                layers='heads')

def color_splash(image, mask):
    """Apply color splash effect.
    image: RGB image [height, width, 3]
    mask: instance segmentation mask [height, width, instance count]

    Returns result image.
    """
    # Make a grayscale copy of the image. The grayscale copy still
    # has 3 RGB channels, though.
    gray = skimage.color.gray2rgb(skimage.color.rgb2gray(image)) * 255
    # Copy color pixels from the original color image where mask is set
    if mask.shape[-1] > 0:
        # We're treating all instances as one, so collapse the mask into one layer
        mask = (np.sum(mask, -1, keepdims=True) >= 1)
        splash = np.where(mask, image, gray).astype(np.uint8)
    else:
        splash = gray.astype(np.uint8)
    return splash

def color_show(image, mask):
    """Apply color splash effect.
    image: RGB image [height, width, 3]
    mask: instance segmentation mask [height, width, instance count]

    Returns result image.
    """
    # Make a grayscale copy of the image. The grayscale copy still
    # has 3 RGB channels, though.
    gray = skimage.color.gray2rgb(skimage.color.rgb2gray(image)) * 255
    # Copy color pixels from the original color image where mask is set
    if mask.shape[-1] > 0:
        # We're treating all instances as one, so collapse the mask into one layer
        mask = (np.sum(mask, -1, keepdims=True) >= 1)
        splash = np.where(mask, image, gray).astype(np.uint8)
    else:
        splash = gray.astype(np.uint8)
    return splash

def detect_and_color_splash(model, image_path=None, video_path=None):
    assert image_path or video_path

    # Image or video?
    if image_path:
        # Run model detection and generate the color splash effect
        print("Running on {}".format(args.image))
        # Read image
        image = skimage.io.imread(args.image)
        # Detect objects
        r = model.detect([image], verbose=1)[0]
        # Color splash
        splash = color_splash(image, r['masks'])
        # Save output
        file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
        skimage.io.imsave(file_name, splash)
    elif video_path:
        import cv2
        # Video capture
        vcapture = cv2.VideoCapture(video_path)
        width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
        fps = vcapture.get(cv2.CAP_PROP_FPS)

        # Define codec and create video writer
        file_name = "splash_{:%Y%m%dT%H%M%S}.avi".format(datetime.datetime.now())
        vwriter = cv2.VideoWriter(file_name,
                                  cv2.VideoWriter_fourcc(*'MJPG'),
                                  fps, (width, height))

        count = 0
        success = True
        while success:
            print("frame: ", count)
            # Read next image
            success, image = vcapture.read()
            if success:
                # OpenCV returns images as BGR, convert to RGB
                image = image[..., ::-1]
                # Detect objects
                r = model.detect([image], verbose=0)[0]
                # Color splash
                splash = color_splash(image, r['masks'])
                # RGB -> BGR to save image to video
                splash = splash[..., ::-1]
                # Add image to video writer
                vwriter.write(splash)
                count += 1
        vwriter.release()
    print("Saved to ", file_name)

def get_ax(rows=1, cols=1, size=16): # bai
    """Return a Matplotlib Axes array to be used in
    all visualizations in the notebook. Provide a
    central point to control graph sizes.

    Adjust the size attribute to control how big to render images
    """
    _, ax = plt.subplots(rows, cols, figsize=(size * cols, size * rows))
    return ax

def detect_and_show(model, image_path=None, video_path=None): #bai
    import visualize_cv2  # bai
    assert image_path or video_path

    # Image or video?
    if image_path:
        # Validation dataset
        dataset = sicDataset()  # bai
        dataset.load_sic(args.dataset, "val")  # bai

        dataset.prepare()

        print("Images: {}\nClasses: {}".format(len(dataset.image_ids), dataset.class_names))
        # Run model detection and generate the color splash effect
        print("Running on {}".format(args.image))
        # Read image
        image = skimage.io.imread(args.image)
        # Detect objects
        results = model.detect([image], verbose=1)
        r = results[0]
        # Save output
        file_name = "detected_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())

        N = r['rois'].shape[0]

#        filter_classs_names = ['BPD', 'TSD', 'TED']
        filter_classs_names = ['bpd', 'tsd', 'ted']
        visualize_cv2.save_image(image, file_name, r['rois'], r['masks'],
                             r['class_ids'], r['scores'], dataset.class_names,
                             filter_classs_names, scores_thresh=0.7, mode=0)
    elif video_path:
        import cv2
        # Video capture
        vcapture = cv2.VideoCapture(video_path)
        width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
        fps = vcapture.get(cv2.CAP_PROP_FPS)

        # Define codec and create video writer
        file_name = "splash_{:%Y%m%dT%H%M%S}.avi".format(datetime.datetime.now())
        vwriter = cv2.VideoWriter(file_name,
                                  cv2.VideoWriter_fourcc(*'MJPG'),
                                  fps, (width, height))

        count = 0
        success = True
        while success:
            print("frame: ", count)
            # Read next image
            success, image = vcapture.read()
            if success:
                # OpenCV returns images as BGR, convert to RGB
                image = image[..., ::-1]
                # Detect objects
                r = model.detect([image], verbose=0)[0]
                # Color splash
                splash = color_show(image, r['masks'])
                # RGB -> BGR to save image to video
                splash = splash[..., ::-1]
                # Add image to video writer
                vwriter.write(splash)
                count += 1
        vwriter.release()
    print("Saved to ", file_name)
############################################################
#  Training
############################################################

if __name__ == '__main__':
    import argparse

    # Parse command line arguments
    parser = argparse.ArgumentParser(
        description='Train Mask R-CNN to detect sic.')
    parser.add_argument("command",
                        metavar="<command>",
                        help="'train' or 'splash'")
    parser.add_argument('--dataset', required=False,
                        metavar="/path/to/balloon/dataset/",
                        help='Directory of the sic dataset')
    parser.add_argument('--weights', required=True,
                        metavar="/path/to/weights.h5",
                        help="Path to weights .h5 file or 'coco'")
    parser.add_argument('--logs', required=False,
                        default=DEFAULT_LOGS_DIR,
                        metavar="/path/to/logs/",
                        help='Logs and checkpoints directory (default=logs/)')
    parser.add_argument('--image', required=False,
                        metavar="path or URL to image",
                        help='Image to apply the color splash effect on')
    parser.add_argument('--video', required=False,
                        metavar="path or URL to video",
                        help='Video to apply the color splash effect on')
    args = parser.parse_args()

    # Validate arguments
    if args.command == "train":
        assert args.dataset, "Argument --dataset is required for training"
    elif args.command == "splash":
        assert args.image or args.video,\
               "Provide --image or --video to apply color splash"
    elif args.command == "show": # bai
        assert args.image or args.video,\
               "Provide --image or --video to show detected result"

    print("Weights: ", args.weights)
    print("Dataset: ", args.dataset)
    print("Logs: ", args.logs)

    # Configurations
    if args.command == "train":
        config = sicConfig() # bai
    else:
        class InferenceConfig(sicConfig): # bai
            # Set batch size to 1 since we'll be running inference on
            # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
            GPU_COUNT = 1
            IMAGES_PER_GPU = 1
        config = InferenceConfig()
    config.display()

    # Create model
    if args.command == "train":
        model = modellib.MaskRCNN(mode="training", config=config,
                                  model_dir=args.logs)
    else:
        model = modellib.MaskRCNN(mode="inference", config=config,
                                  model_dir=args.logs)

    # Select weights file to load
    if args.weights.lower() == "coco":
        weights_path = COCO_WEIGHTS_PATH
        # Download weights file
        if not os.path.exists(weights_path):
            utils.download_trained_weights(weights_path)
    elif args.weights.lower() == "last":
        # Find last trained weights
        weights_path = model.find_last()
    elif args.weights.lower() == "imagenet":
        # Start from ImageNet trained weights
        weights_path = model.get_imagenet_weights()
    else:
        weights_path = args.weights

    # Load weights
    print("Loading weights ", weights_path)
    if args.weights.lower() == "coco":
        # Exclude the last layers because they require a matching
        # number of classes
        model.load_weights(weights_path, by_name=True, exclude=[
            "mrcnn_class_logits", "mrcnn_bbox_fc",
            "mrcnn_bbox", "mrcnn_mask"])
    else:
        model.load_weights(weights_path, by_name=True)

    # Train or evaluate
    if args.command == "train":
        train(model)
    elif args.command == "splash":
        detect_and_color_splash(model, image_path=args.image,
                                video_path=args.video)
    elif args.command == "show": # bai
        detect_and_show(model, image_path=args.image,
                                video_path=args.video)
    else:
        print("'{}' is not recognized. "
              "Use 'train' or 'splash'".format(args.command))


# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for query_utils."""

import itertools
import math
import operator

import mock

from google.appengine.ext import ndb

from absl.testing import absltest
from upvote.gae.shared.common import basetest
from upvote.gae.shared.common import query_utils
from upvote.shared import constants

class TestModel(ndb.Model):
  foo = ndb.StringProperty()
  bar = ndb.IntegerProperty()

def CreateEntity(foo='foo', bar=0):
  entity = TestModel(foo=foo, bar=bar)
  entity.put()
  return entity

def CreateEntities(count, **kwargs):
  return [CreateEntity(**kwargs) for _ in xrange(count)]

_GLOBAL_CBK_MOCK = mock.MagicMock()

def CallMock(*args, **kwargs):
  _GLOBAL_CBK_MOCK(*args, **kwargs)

def GetKey(key):
  return key.get()

def ReturnFoo(entity):
  return entity.foo

def ReturnBar(entity):
  return entity.bar

class QueryUtilsTest(basetest.UpvoteTestCase):

  def setUp(self):
    super(QueryUtilsTest, self).setUp()

  def tearDown(self):
    super(QueryUtilsTest, self).tearDown()
    _GLOBAL_CBK_MOCK.reset_mock()

  def testPaginatedMap(self):
    CreateEntities(3)
    result = query_utils.PaginatedMap(TestModel.query(), ReturnFoo)
    self.assertEqual(3, len(result))
    self.assertEqual('foo', result[0])

  def testPaginatedMap_TwoPages(self):
    CreateEntities(3)
    cbk_mock = mock.MagicMock()

    result = query_utils.PaginatedMap(TestModel.query(), cbk_mock, page_size=2)
    self.assertEqual(3, len(result))
    self.assertEqual(3, cbk_mock.call_count)

  def testPaginatedMap_ThreePages(self):
    CreateEntities(3)
    cbk_mock = mock.MagicMock()

    result = query_utils.PaginatedMap(TestModel.query(), cbk_mock, page_size=1)
    self.assertEqual(3, len(result))
    self.assertEqual(3, cbk_mock.call_count)

  def testPaginatedMap_QueryOptions(self):
    CreateEntities(3)

    result = query_utils.PaginatedMap(TestModel.query(), GetKey, keys_only=True)

    self.assertEqual(3, len(result))
    self.assertTrue(all(isinstance(entity, TestModel) for entity in result))

  def testPaginatedMapReduce(self):
    CreateEntities(3)
    map_cbk_mock = mock.MagicMock()
    reduce_cbk_mock = mock.MagicMock()
    reduce_cbk_mock.return_value = None

    result = query_utils.PaginatedMapReduce(
        TestModel.query(), map_cbk_mock, reduce_cbk_mock, page_size=1)
    self.assertIsNone(result)
    self.assertEqual(3, map_cbk_mock.call_count)
    self.assertEqual(3, reduce_cbk_mock.call_count)

  def testPaginatedMapReduce_Initial(self):
    CreateEntities(3, bar=1)

    result = query_utils.PaginatedMapReduce(
        TestModel.query(), ReturnBar, operator.add, initial=5, page_size=1)
    self.assertEqual(5 + 3, result)

  def testPaginatedFetch(self):
    CreateEntities(3)
    result = query_utils.PaginatedFetch(TestModel.query(), page_size=2)
    map_result = query_utils.PaginatedMap(TestModel.query(), None, page_size=2)
    self.assertEqual(3, len(result))
    self.assertEqual(map_result, result)

  def testPaginate(self):

    page_size = 10
    for entity_count in xrange(50):

      # Create some number of entities.
      CreateEntities(entity_count)

      # Verify that we get the expected number of pages.
      pages = list(query_utils.Paginate(TestModel.query(), page_size=page_size))
      expected_page_count = int(math.ceil(float(entity_count) / page_size))
      self.assertEqual(expected_page_count, len(pages))

      # Verify that we get the expected number of entities.
      entities = list(itertools.chain(*pages))
      self.assertEqual(entity_count, len(entities))

      # Delete everything.
      for entity in entities:
        entity.key.delete()

  def testQueuedPaginatedApply(self):
    CreateEntities(3)
    query_utils.QueuedPaginatedApply(TestModel.query(), CallMock, page_size=1)

    for _ in xrange(4):
      self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 1)
      self.RunDeferredTasks()

    self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 0)

    self.assertEqual(3, _GLOBAL_CBK_MOCK.call_count)

  def testQueuedPaginatedApply_ExtraArgs(self):
    CreateEntities(3)
    query_utils.QueuedPaginatedApply(
        TestModel.query(), CallMock, extra_args=['a', 'b'],
        extra_kwargs={'c': 'c'})

    for _ in xrange(2):
      self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 1)
      self.RunDeferredTasks()

    self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 0)

    self.assertTrue(_GLOBAL_CBK_MOCK.called_with('foo', 'a', 'b', c='c'))

  def testQueuedPaginatedApply_Transform(self):
    CreateEntities(3)
    query_utils.QueuedPaginatedApply(
        TestModel.query(), CallMock, pre_queue_callback=ReturnFoo, page_size=1)

    for _ in xrange(4):
      self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 1)
      self.RunDeferredTasks()

    self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 0)

    self.assertTrue(_GLOBAL_CBK_MOCK.called_with('foo'))

  def testQueuedPaginatedApply_DifferentQueue(self):
    CreateEntities(3)
    query_utils.QueuedPaginatedApply(TestModel.query(), CallMock, queue='foo')

    for _ in xrange(2):
      self.assertTaskCount('foo', 1)
      self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 0)
      self.RunDeferredTasks('foo')

    self.assertTaskCount('foo', 0)

    self.assertEqual(3, _GLOBAL_CBK_MOCK.call_count)

  def testQueuedPaginatedApply_QueryOptions(self):
    entities = CreateEntities(3)
    query_utils.QueuedPaginatedApply(
        TestModel.query(), CallMock, keys_only=True)

    for _ in xrange(2):
      self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 1)
      self.RunDeferredTasks()

    self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 0)

    self.assertEqual(3, _GLOBAL_CBK_MOCK.call_count)
    self.assertTrue(_GLOBAL_CBK_MOCK.called_with(entities[0].key))

  def testQueuedPaginatedBatchApply(self):
    entities = CreateEntities(3)
    query_utils.QueuedPaginatedBatchApply(
        TestModel.query(), CallMock, page_size=2)

    for _ in xrange(3):
      self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 1)
      self.RunDeferredTasks()

    self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 0)

    self.assertTrue(_GLOBAL_CBK_MOCK.called_with(entities[:2]))
    self.assertTrue(_GLOBAL_CBK_MOCK.called_with(entities[2:]))
    self.assertEqual(2, _GLOBAL_CBK_MOCK.call_count)

  def testQueuedPaginatedBatchApply_ExtraArgs(self):
    entities = CreateEntities(1)
    query_utils.QueuedPaginatedBatchApply(
        TestModel.query(), CallMock, extra_args=['a', 'b'],
        extra_kwargs={'c': 'c'})

    for _ in xrange(2):
      self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 1)
      self.RunDeferredTasks()

    self.assertTaskCount(constants.TASK_QUEUE.DEFAULT, 0)

    self.assertTrue(_GLOBAL_CBK_MOCK.called_with(entities, 'a', 'b', c='c'))

if __name__ == '__main__':
  absltest.main()


import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc

rc('font', family='serif')

def calcAngles(theta1):
	'''Takes input link angle, performs 4 bar kinematics calcs and returns angles of all links'''
	theta = np.zeros(4)
	theta[0] = 0
	theta[1] = theta1

	Alpha = 2.*L[0]*L[3] * np.cos(theta[0]) - 2.*L[1]*L[3]*np.cos(theta[1])
	Beta  = 2.*L[0]*L[3] * np.sin(theta[0]) - 2.*L[1]*L[3]*np.sin(theta[1])
	Gamma = (L[0]**2 + L[1]**2 + L[3]**2 - L[2]**2 - 2.*L[0]*L[1]*np.cos(theta[0]) * np.cos(theta[1])
																	   - 2.*L[0]*L[1]*np.sin(theta[0]) * np.sin(theta[1]))
	Delta = np.arctan2(Beta,Alpha)

	theta[3] = Delta + np.arccos(-1*Gamma/np.sqrt(Alpha**2 + Beta**2))
	theta[2] = (np.arctan2(L[0]*np.sin(theta[0]) + L[3]*np.sin(theta[3]) - L[1]*np.sin(theta[1]),
						L[0]*np.cos(theta[0]) + L[3]*np.cos(theta[3]) - L[1]*np.cos(theta[1])))
	return theta

def calcPos(theta):
	Pos = np.zeros((5,2))
	Opos = np.zeros((1,2))
	'''Returns x,y coordinates of Joints O,A,B,C and Foot Connection of four bar given position of point O'''
	Pos[0,:] =  Opos
	Pos[1,:] =  Opos + np.array([L[1]*np.cos(theta[1]), L[1]*np.sin(theta[1])]) #A
	Pos[2,:] = (Opos + np.array([L[0]*np.cos(theta[0]), L[0]*np.sin(theta[0])])
							   + np.array([L[3]*np.cos(theta[3]), L[3]*np.sin(theta[3])])) #B
	Pos[3,:] =  Opos + np.array([L[0]*np.cos(theta[0]), L[0]*np.sin(theta[0])]) #C
	Pos[4,:] = (Opos + np.array([L[1]*np.cos(theta[1]), L[1]*np.sin(theta[1])])
							   + np.array([L[4]*np.cos(theta[2] + np.pi), L[4]*np.sin(theta[2] + np.pi)]))   #F
	return Pos

def calcAngSpeeds(theta):
	'''Solves system of equations and returns angular speeds given input link speed omega1'''
	omega = np.zeros(4)
	omega[1] = 70.
	omega[0] = 0.

	A = np.array([[-1*L[3]*np.sin(theta[3]),    L[2]*np.sin(theta[2])],
					 [L[3]*np.cos(theta[3]), -1*L[2]*np.cos(theta[2])]])
	b = ( np.array([[-1*L[1]*np.sin(theta[1])],
					   [L[1]*np.cos(theta[1])]]) * omega[1]
		+ np.array([[-1*L[0]*np.sin(theta[0])],
					   [L[0]*np.cos(theta[0])]]) * omega[0])
	omega32 = np.linalg.solve(A,b)
	omega[3] = omega32[0]
	omega[2] = omega32[1]

	return omega

def calcSpeed(theta,omega):
	'''Returns x,y speeds of Joints O,A,B,C and Foot Connection of four bar given speed of point O'''
	jointSpeed = np.zeros((5,2))
	Ospeed = np.zeros((1,2))
	jointSpeed[0,:] = Ospeed
	jointSpeed[1,:] = Ospeed + np.array([-1*L[1] * np.sin(theta[1]), L[1] * np.cos(theta[1])])*omega[1] #A
	jointSpeed[2,:] = Ospeed + np.array([-1*L[0] * np.sin(theta[0]), L[0] * np.cos(theta[0])])*omega[0]+ np.array([-1*L[3] * np.sin(theta[3]), L[3] * np.cos(theta[3])])*omega[3] #B
	jointSpeed[3,:] = Ospeed + np.array([-1*L[0] * np.sin(theta[0]), L[0] * np.cos(theta[0])])*omega[0]#C
	jointSpeed[4,:] = Ospeed + (np.array([-1*L[1]*np.sin(theta[1]),  L[1]*np.cos(theta[1])		  ])*omega[1]
								  +  np.array([-1*L[4]*np.sin(theta[2] + np.pi), L[4]*np.cos(theta[2] + np.pi)])*omega[2]) #F
	return jointSpeed

L = [0.0615, 0.0218, 0.0748, 0.0468, 0.0624]
i = 0

plt.figure(num = 1, figsize = (3,2))
plt.axes(frameon=False)
plt.gca().axes.get_xaxis().set_visible(False)
plt.gca().axes.get_yaxis().set_visible(False)
footPts = np.empty((360,2))
footSpeed = np.empty((360,2))
for theta1 in np.linspace(0,2*np.pi,360):
	theta = calcAngles(theta1)
	omega = calcAngSpeeds(theta)
	O, A, B, C, F  = calcPos(theta)
	_, _, _, _, Fs = calcSpeed(theta,omega)
	footPts[i,:] = F
	footSpeed[i,:] = Fs

	if i == 180 or i == 180+36 or i == 180 + 72:
		legPts = np.array([O, A, F, B, C])
		plt.plot(legPts[:,0],legPts[:,1],'k',lw = 0.3, color = str((252. - i)/90.))
	i += 1

plt.plot(footPts[:,0],footPts[:,1],lw = 1.5)
r = 0.0218
theta = np.linspace(0,2*np.pi,360)
plt.plot(r*np.cos(theta),r*np.sin(theta),lw = .3)
plt.axis('equal')
x1, x2, y1,y2 = plt.axis()
plt.savefig('leg.pdf', bbox_inches='tight')

plt.figure(num = 2, figsize = (3,2))
theta = np.linspace(0,2*np.pi,360)
plt.axes(frameon=False)
plt.gca().axes.set_xticks([])
plt.gca().axes.set_yticks([])
plt.xlabel('y-velocity')
plt.ylabel('y-position')
plt.plot(footSpeed[:,1],footPts[:,1],'r',lw = 1.5)
#plt.axis('equal')
plt.savefig('legspeed.pdf', bbox_inches='tight')

plt.figure(num = 3, figsize = (3,2))
theta = np.linspace(0,2*np.pi,360)
plt.axes(frameon=False)
plt.gca().axes.get_xaxis().set_visible(False)
plt.gca().axes.get_yaxis().set_visible(False)

r = .026
i =0
for theta1 in np.linspace(0,2*np.pi,360):
	if i == 180 or i == 180+36 or i == 180 + 72:
		x = r*np.cos(theta1)
		y = r*np.sin(theta1)
		plt.plot([0,x],[0,y],'k',lw = 0.5, color = str((252. - i)/90.))
	i += 1

plt.plot(r*np.cos(theta),r*np.sin(theta),lw = 2)
plt.axis('equal')
plt.savefig('leg2.pdf', bbox_inches='tight')

plt.figure(num = 4, figsize = (3,2))
theta = np.linspace(0,2*np.pi,360)
plt.axes(frameon=False)
plt.xlabel('y-velocity')
plt.ylabel('y-position')
plt.gca().axes.set_xticks([])
plt.gca().axes.set_yticks([])

plt.plot(70*r*np.cos(theta),r*np.sin(theta),'r',lw = 1.5)
#plt.axis('equal')
plt.savefig('leg2speed.pdf', bbox_inches='tight')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

'''
    Author: Weiliang Luo
    Student No.: 1900011804
    Date changed: 2021.5.8
    Python Version: Anaconda3 (Python 3.8.6)
'''
# Imports
# std libs
import argparse

# 3rd Party libs

# my modules

def parseArgs():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--gpu',
        type=bool,
        default=True,
        help="try to use cuda"
    )
    parser.add_argument(
        '--fgsm',
        type=bool,
        default=False,
        help="run fgsm attack"
    )
    parser.add_argument(
        '--draw',
        type=bool,
        default=False,
        help="draw fgsm adversial examples"
    )
    parser.add_argument(
        '--pgd',
        type=bool,
        default=False,
        help="run pgd attack"
    )
    parser.add_argument(
        '--black',
        type=bool,
        default=False,
        help="run blackbox attack"
    )
    parser.add_argument(
        '--randinit',
        type=bool,
        default=False,
        help="random initialize in pgd"
    )
    parser.add_argument(
        '--pgditer',
        type=int,
        default=10,
        help="iteration number in pgd"
    )
    flags, unparsed = parser.parse_known_args()
    return flags


# import tensorflow as tf
import tensorflow.compat.v1 as tf

def xdeepfm_model_fn(features, labels, mode, params):
    deep_columns     = params['deep_columns']
    deep_fields_size = params['deep_fields_size']
    org_emb_size     = params['embedding_dim']
    wide_columns     = params['wide_columns']
    wide_fields_size = params['wide_fields_size']
    deep_input_layer = tf.feature_column.input_layer(features=features, feature_columns=deep_columns)

    with tf.name_scope('wide'):
        wide_input_layer = tf.feature_column.input_layer(features=features, feature_columns=wide_columns)
        wide_output_layer = tf.layers.dense(inputs=wide_input_layer, units=1, activation=None, use_bias=True)

    with tf.name_scope('deep'):
        d_layer_1 = tf.layers.dense(inputs=deep_input_layer, units=50, activation=tf.nn.relu, use_bias=True)
        bn_layer_1 = tf.layers.batch_normalization(inputs=d_layer_1, axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True)
        deep_output_layer = tf.layers.dense(inputs=bn_layer_1, units=40, activation=tf.nn.relu, use_bias=True)

    with tf.name_scope('cin'):
        deep_feat_emb = tf.reshape(deep_input_layer, [-1, deep_fields_size, org_emb_size])
        cross_layer_sizes = [64, 64, 32]
        dim = org_emb_size
        final_len = 0
        field_nums = [int(deep_fields_size)]
        cin_layers = [deep_feat_emb]
        final_result = []
        split_tensor0 = tf.split(cin_layers[0], dim * [1], 2)
        for idx, layer_size in enumerate(cross_layer_sizes):
            split_tensor = tf.split(cin_layers[-1], dim * [1], 2)
            dot_result_m = tf.matmul(split_tensor0, split_tensor, transpose_b=True)
            dot_result_o = tf.reshape(dot_result_m, shape=[dim, -1, field_nums[0] * field_nums[-1]])
            dot_result = tf.transpose(dot_result_o, perm=[1, 0, 2])

            conv_filters = tf.get_variable(name='f_'+str(idx),
                                           shape=[1, field_nums[0] * field_nums[-1], layer_size],
                                           dtype=tf.float32)
            curr_out = tf.nn.conv1d(dot_result, filters=conv_filters, stride=1, padding='VALID')
            curr_out = tf.nn.relu(curr_out)
            curr_out = tf.transpose(curr_out, perm=[0, 2, 1])

            dirrect_connect = curr_out
            next_hidden = curr_out
            final_len += int(layer_size)
            field_nums.append(int(layer_size))

            final_result.append(dirrect_connect)
            cin_layers.append(next_hidden)
        result = tf.concat(final_result, axis=1)
        result = tf.reduce_sum(result, -1)

    with tf.name_scope('concat'):
        m_layer = tf.concat([wide_output_layer, deep_output_layer, result], 1)
        o_layer = tf.layers.dense(inputs=m_layer, units=1, activation=None, use_bias=True)

    with tf.name_scope('logit'):
        o_prob = tf.nn.sigmoid(o_layer)
        predictions = tf.cast((o_prob > 0.5), tf.float32)

    if mode == tf.estimator.ModeKeys.PREDICT:
        predictions = {
            'probabilities':    o_prob,
            'label':            predictions
        }
        return tf.estimator.EstimatorSpec(mode, predictions=predictions)

    loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=o_layer))

    if mode == tf.estimator.ModeKeys.TRAIN:
        if   params['optimizer'] == 'adam':
            optimizer = tf.train.AdamOptimizer(learning_rate=params['learning_rate'], beta1=0.9, beta2=0.999, epsilon=1e-8)
        elif params['optimizer'] == 'adagrad':
            optimizer = tf.train.AdagradOptimizer(learning_rate=params['learning_rate'], initial_accumulator_value=1e-8)
        elif params['optimizer'] == 'momentum':
            optimizer = tf.train.MomentumOptimizer(learning_rate=params['learning_rate'], momentum=0.95)
        elif params['optimizer'] == 'ftrl':
            optimizer = tf.train.FtrlOptimizer(learning_rate=params['learning_rate'])
        else:
            optimizer = tf.train.GradientDescentOptimizer(learning_rate=params['learning_rate'])
        train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
        return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)

    if mode == tf.estimator.ModeKeys.EVAL:
        accuracy = tf.metrics.accuracy(labels, predictions)
        auc = tf.metrics.auc(labels, predictions)
        my_metrics = {
            'accuracy': tf.metrics.accuracy(labels, predictions),
            'auc':      tf.metrics.auc(labels,predictions)
        }
        tf.summary.scalar('accuracy', accuracy[1])
        tf.summary.scalar('auc', auc[1])
        return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=my_metrics)


# -*- coding: utf-8 -*-
"""
Created on Mon May 13 14:37:13 2019

@author: UARTman
"""

def make_turn(field):
    variants={}
    r = []
    for i in range(len(field)):
        for j in range(len(field)):
            if field[i][j] == 0:
                variants[(i, j)] = 0
    for q in variants:
        for i in range(len(field)):
            if i != q[0] and field[i][q[1]] == 0:
                variants[q] += 1
            if i != q[1] and field[q[0]][i] == 0:
                variants[q] += 1
    for i in variants:
        r.append(variants[i])
    for i in variants:
        if variants[i] == max(r):
            return [i[0], i[1], 4]
    return [-1, -1, 4]


from distutils.core import setup, Extension
from Cython.Build import cythonize
import os

#project_name = "gutzwiller"
omp = False

cpp_dir = os.sep.join(os.getcwd().split(os.sep)[:-1])

def cpp_path(cpp_file):
    return os.path.join(cpp_dir, cpp_file)

source_files = []
for file in os.listdir(cpp_dir):
    if file.endswith(".cpp") and file != "main.cpp":
        source_files.append(cpp_path(file))

#armadillo_include_dir = os.path.join(os.path.expanduser("~"), "Armadillo", "usr", "include")

#openblas_include_dir = os.path.join(os.path.expanduser("~"), "OpenBLAS", "include")
# = os.path.join(os.path.expanduser("~"), "OpenBLAS", "lib")

include_dirs = [cpp_dir]#, openblas_include_dir, armadillo_include_dir]
library_dirs = []#[openblas_library_dir]

extra_compile_args = ["-std=c++11", "-O3", "-march=native"] #, "-DARMA_DONT_USE_WRAPPER"]
if omp:
    extra_compile_args.append('-fopenmp')
extra_link_args = ['-fopenmp'] if omp else None

setup(ext_modules = cythonize(Extension(
		"Gutzwiller",
        language="c++",
        sources=["Gutzwiller.pyx"] + source_files,
        include_dirs=include_dirs,
        library_dirs=library_dirs,
        runtime_library_dirs=library_dirs,
        libraries=["m", "gsl", "armadillo"],# "openblas", "lapack"],
        extra_compile_args=extra_compile_args,
        extra_link_args=extra_link_args
      ), compiler_directives={'language_level': 3}
))