# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/garl_gym/scenarios/simple_population_dynamics_ga.py
# Compiled at: 2019-06-10 11:29:04
# Size of source mod 2**32: 26191 bytes
import os, sys, random
from PIL import Image
import numpy as np, matplotlib.pyplot as plt
from cv2 import VideoWriter, imread, resize
import cv2
from copy import deepcopy
from garl_gym.base import BaseEnv
import multiprocessing as mp, gc
from garl_gym.core import DiscreteWorld, Agent

class SimplePopulationDynamicsGA(BaseEnv):
    __doc__ = '\n    args:\n        - height\n        - width\n        - batch_size\n        - view_args\n        - agent_numbee\n        - num_actions ... not necessary(add flag?)\n        - damage_per_step\n\n    In the future, we define the attack ?\n    If continuous, then size and speed?\n    '

    def __init__(self, args):
        self.args = args
        self.h = args.height
        self.w = args.width
        self.batch_size = args.batch_size
        if hasattr(args, 'view_args'):
            self.view_args = args.view_args
        else:
            self.view_args = None
        self.agent_num = args.predator_num
        self.predator_num = args.predator_num
        self.prey_num = args.prey_num
        self.action_num = args.num_actions
        self.vision_width = args.vision_width
        self.vision_height = args.vision_height
        self.map = np.zeros((self.h, self.w), dtype=(np.int32))
        self.food_map = np.zeros((self.h, self.w), dtype=(np.int32))
        self.property = {}
        self.killed = []
        self.max_health = args.max_health
        self.min_health = args.min_health
        self.max_id = 1
        self.rewards = None
        self.max_view_size = None
        self.min_view_size = None
        self._init_property()
        self.max_hunt_square = args.max_hunt_square
        self.max_speed = args.max_speed
        self.max_ = args.max_crossover
        self.timestep = 0
        self.num_food = 0
        self.obs_type = args.obs_type
        self.agent_embeddings = {}
        self.agent_emb_dim = args.agent_emb_dim
        self.cpu_cores = args.cpu_cores

    @property
    def agents(self):
        return {**(self.predators), **(self.preys)}

    def make_world(self, wall_prob=0, wall_seed=10, food_prob=0.1, food_seed=10):
        self.gen_wall(wall_prob, wall_seed)
        self.gen_food(food_prob, food_seed)
        predators = {}
        preys = {}
        agents = [Agent() for _ in range(self.predator_num + self.prey_num)]
        empty_cells_ind = np.where(self.map == 0)
        perm = np.random.permutation(range(len(empty_cells_ind[0])))
        for i, agent in enumerate(agents):
            agent.name = 'agent {:d}'.format(i + 1)
            health = np.random.uniform(self.min_health, self.max_health)
            agent.health = health
            agent.original_health = health
            agent.birth_time = self.timestep
            if i < self.predator_num:
                agent.predator = True
                agent.id = self.max_id
                agent.speed = 1
                agent.hunt_square = self.max_hunt_square
                agent.property = [self._gen_power(i + 1), [0, 0, 1]]
            else:
                agent.predator = False
                agent.id = i + 1
                agent.property = [self._gen_power(i + 1), [1, 0, 0]]
            new_embedding = np.random.normal(size=[self.agent_emb_dim])
            self.agent_embeddings[agent.id] = new_embedding
            x = empty_cells_ind[0][perm[i]]
            y = empty_cells_ind[1][perm[i]]
            self.map[x][y] = self.max_id
            agent.pos = (x, y)
            self.max_id += 1
            if agent.predator:
                predators[agent.id] = agent
            else:
                preys[agent.id] = agent
            self.predators = predators
            self.preys = preys

    def gen_food(self, prob=0.1, seed=10):
        for i in range(self.h):
            for j in range(self.w):
                food_prob = np.random.rand()
                if food_prob < prob and self.map[i][j] != -1 and self.food_map[i][j] == 0:
                    self.food_map[i][j] = -2
                    self.num_food += 1

    def gen_wall(self, prob=0, seed=10):
        if prob == 0:
            return
        np.random.seed(seed)
        for i in range(self.h):
            for j in range(self.w):
                if i == 0 or i == self.h - 1 or j == 0 or j == self.w - 1:
                    self.map[i][j] = -1
                else:
                    wall_prob = np.random.rand()
                    if wall_prob < prob:
                        self.map[i][j] = -1

    def _init_property(self):
        self.property[-3] = [
         1, [1, 0, 0]]
        self.property[-2] = [1, [0, 1, 0]]
        self.property[-1] = [1, [0.411, 0.411, 0.411]]
        self.property[0] = [1, [0, 0, 0]]

    def _gen_power(self, cnt):

        def max_view_size(view_size1, view_size2):
            view_size_area1 = (2 * view_size1[0] + 1) * (view_size1[1] + 1)
            view_size_area2 = (2 * view_size2[0] + 1) * (view_size2[1] + 1)
            if view_size_area1 > view_size_area2:
                return view_size1
            else:
                return view_size2

        def min_view_size(view_size1, view_size2):
            view_size_area1 = (2 * view_size1[0] + 1) * (view_size1[1] + 1)
            view_size_area2 = (2 * view_size2[0] + 1) * (view_size2[1] + 1)
            if view_size_area1 < view_size_area2:
                return view_size1
            else:
                return view_size2

        cur = 0
        if self.view_args is None:
            return [
             5, 5, 0]
        for k in self.view_args:
            k = [int(x) for x in k.split('-')]
            assert len(k) == 4
            num, power_list = k[0], k[1:]
            if self.max_view_size is None:
                self.max_view_size = power_list
            else:
                self.max_view_size = max_view_size(self.max_view_size, power_list)
            if self.min_view_size is None:
                self.min_view_size = power_list
            else:
                self.min_view_size = min_view_size(self.min_view_size, power_list)
            cur += num
            if cnt <= cur:
                return power_list

    def increase_food(self, prob):
        num = max(1, int(self.num_food * prob))
        ind = np.where(self.food_map == 0)
        num = min(num, len(ind[0]))
        perm = np.random.permutation(np.arange(len(ind[0])))
        for i in range(num):
            x = ind[0][perm[i]]
            y = ind[1][perm[i]]
            if self.map[x][y] != -1 and self.food_map[x][y] == 0:
                self.food_map[x][y] = -2
                self.num_food += 1

    def crossover_predator(self, crossover_scope=3, crossover_rate=0.001):
        ind = np.where(self.map == 0)
        perm = np.random.permutation(np.arange(len(ind[0])))
        index = 0
        for predator in list(self.predators.values()):
            x, y = predator.pos
            local_map = self.map[x - crossover_scope // 2:x - crossover_scope // 2 + crossover_scope, y - crossover_scope // 2:y - crossover_scope // 2 + crossover_scope]
            agent_indice = np.where(local_map > 0)
            if not len(agent_indice[0]) == 0:
                if predator.crossover:
                    pass
                else:
                    for local_x, local_y in zip(agent_indice[0], agent_indice[1]):
                        candidate_id = local_map[(local_x, local_y)]
                        candidate_agent = self.agents[candidate_id]
                        if candidate_agent.predator and not candidate_agent.crossover and predator.id != candidate_agent.id and np.random.rand() < crossover_rate:
                            candidate_agent.crossover = True
                            predator.crossover = True
                            child = Agent()
                            child.id = self.max_id
                            self.max_id += 1
                            new_embedding = np.random.normal(size=[self.agent_emb_dim])
                            self.agent_embeddings[child.id] = new_embedding
                            child.spped = None
                            child.predator = True
                            child.health = 1
                            child.hunt_square = self.max_hunt_square
                            child.property = [self._gen_power(child.id), [0, 0, 1]]
                            x = ind[0][perm[index]]
                            y = ind[1][perm[index]]
                            index += 1
                            self.map[x][y] = child.id
                            child.pos = (x, y)
                            self.predators[child.id] = child
                            self.predator_num += 1

    def crossover_prey(self, crossover_scope=3, crossover_rate=0.001):
        ind = np.where(self.map == 0)
        perm = np.random.permutation(np.arange(len(ind[0])))
        index = 0
        for prey in list(self.preys.values()):
            x, y = prey.pos
            left = np.maximum(x - crossover_scope // 2, 0)
            right = np.minimum(x - crossover_scope // 2 + crossover_scope, self.w)
            top = np.maximum(y - crossover_scope // 2, 0)
            bottom = np.minimum(y - crossover_scope // 2 + crossover_scope, self.h)
            local_map = self.map[left:right, top:bottom]
            agent_indice = np.where(local_map > 0)
            if len(agent_indice[0]) == 0:
                if not prey.crossover:
                    continue
            for local_x, local_y in zip(agent_indice[0], agent_indice[1]):
                candidate_id = local_map[(local_x, local_y)]
                candidate_agent = self.agents[candidate_id]
                if not candidate_agent.predator and not candidate_agent.crossover and candidate_agent.id != prey.id and np.random.rand() < crossover_rate:
                    candidate_agent.crossover = True
                    prey.crossover = True
                    child = Agent()
                    child.id = self.max_id
                    self.max_id += 1
                    child.speed = None
                    child.predator = False
                    child.health = 1
                    new_embedding = np.random.normal(size=[self.agent_emb_dim])
                    self.agent_embeddings[child.id] = new_embedding
                    child.hunt_square = self.max_hunt_square
                    child.property = [self._gen_power(child.id), [1, 0, 0]]
                    x = ind[0][perm[index]]
                    y = ind[1][perm[index]]
                    index += 1
                    self.map[x][y] = child.id
                    child.pos = (x, y)
                    self.preys[child.id] = child
                    self.prey_num += 1

    def take_actions(self, actions):
        for id, agent in self.agents.items():
            if agent.predator:
                self._predator_action(agent, actions[id])
                self.decrease_health(agent)
            else:
                self._prey_action(agent, actions[id])

    def _prey_action(self, agent, action):

        def in_board(x, y):
            return not (x < 0 or x >= self.h or y < 0 or y >= self.w) and self.map[x][y] == 0

        x, y = agent.pos
        if action == 0:
            new_x = x - 1
            new_y = y
            if in_board(new_x, new_y):
                agent.pos = (
                 new_x, new_y)
        else:
            if action == 1:
                new_x = x + 1
                new_y = y
                if in_board(new_x, new_y):
                    agent.pos = (
                     new_x, new_y)
            else:
                if action == 2:
                    new_x = x
                    new_y = y - 1
                    if in_board(new_x, new_y):
                        agent.pos = (
                         new_x, new_y)
                else:
                    if action == 3:
                        new_x = x
                        new_y = y + 1
                        if in_board(new_x, new_y):
                            agent.pos = (
                             new_x, new_y)
                    else:
                        print('Wrong action id')
        new_x, new_y = agent.pos
        self.map[x][y] = 0
        self.map[new_x][new_y] = agent.id
        if self.food_map[(new_x, new_y)] == -2:
            self.food_map[(new_x, new_y)] = 0
            agent.health += 0.1
            self.num_food -= 1

    def _predator_action(self, agent, action):

        def in_board(x, y):
            return not (x < 0 or x >= self.h or y < 0 or y >= self.w) and self.map[x][y] == 0

        x, y = agent.pos
        if action == 0:
            new_x = x - 1
            new_y = y
            if in_board(new_x, new_y):
                agent.pos = (
                 new_x, new_y)
        else:
            if action == 1:
                new_x = x + 1
                new_y = y
                if in_board(new_x, new_y):
                    agent.pos = (
                     new_x, new_y)
            else:
                if action == 2:
                    new_x = x
                    new_y = y - 1
                    if in_board(new_x, new_y):
                        agent.pos = (
                         new_x, new_y)
                else:
                    if action == 3:
                        new_x = x
                        new_y = y + 1
                        if in_board(new_x, new_y):
                            agent.pos = (
                             new_x, new_y)
                    else:
                        print('Wrong action id')
        new_x, new_y = agent.pos
        self.map[x][y] = 0
        self.map[new_x][new_y] = agent.id

    def decrease_health(self, agent):
        agent.health -= self.args.damage_per_step

    def increase_health(self, agent):
        if hasattr(self.args, 'health_increase_rate'):
            if self.args.health_increase_rate is not None:
                agent.health += self.args.health_increase_rate
        else:
            agent.health += 1.0

    def dump_image(self, img_name):
        new_w, new_h = self.w * 5, self.h * 5
        img = np.zeros((new_w, new_h, 3), dtype=(np.uint8))
        length = self.args.img_length
        for i in range(self.h):
            for j in range(self.w):
                id = self.map[i][j]
                if self.food_map[i][j] == -2:
                    img[i * length - 1:(i + 1) * length, j * length - 1:(j + 1) * length, :] = 255 * np.array(self.property[(-2)][1])
                elif id == 0:
                    img[i * length - 1:(i + 1) * length, j * length - 1:(j + 1) * length, :] = 255
                elif id == -1:
                    img[i * length - 1:(i + 1) * length, j * length - 1:(j + 1) * length, :] = 255 * np.array(self.property[id][1])
                else:
                    agent = self.agents[id]
                    img[i * length - 1:(i + 1) * length, j * length - 1:(j + 1) * length, :] = 255 * np.array(agent.property[1])

        output_img = Image.fromarray(img, 'RGB')
        output_img.save(img_name)

    def convert_img(self):
        img = np.zeros((self.h, self.w, 3))
        for i in range(self.h):
            for j in range(self.w):
                id = self.map[i][j]
                if self.food_map[i][j] == -2:
                    img[i, j, :] = 255 * np.array(self.property[(-2)][1])
                else:
                    if id == 0:
                        img[i, j, :] = 255
                    else:
                        if id == -1:
                            img[i, j, :] = 255 * np.array(self.property[id][1])
                        else:
                            img[i, j, :] = 255 * np.array(self.property[(-3)][1])

        for predator in self.predators.values():
            x, y = predator.pos
            img[x, y, :] = 255 * np.array(predator.property[1])

        return img

    def get_predator_reward(self, agent):
        reward = 0
        x, y = agent.pos
        left = np.maximum(x - agent.hunt_square // 2, 0)
        right = np.minimum(x - agent.hunt_square // 2 + agent.hunt_square, self.w)
        top = np.maximum(y - agent.hunt_square // 2, 0)
        bottom = np.minimum(y - agent.hunt_square // 2 + agent.hunt_square, self.h)
        local_map = self.map[left:right, top:bottom]
        id_prey_loc = np.where(local_map > 0)
        min_dist = np.inf
        target_prey = None
        killed_id = None
        for local_x, local_y in zip(id_prey_loc[0], id_prey_loc[1]):
            candidate_agent = self.agents[local_map[(local_x, local_y)]]
            if not candidate_agent.predator:
                x_prey, y_prey = candidate_agent.pos
                dist = np.sqrt((x - x_prey) ** 2 + (y - y_prey) ** 2)
                if dist < min_dist:
                    min_dist = dist
                    target_prey = candidate_agent

        if target_prey is not None:
            reward += 1
            target_prey.dead = True
            agent.max_reward += 1
            killed_id = target_prey.id
        if agent.health <= 0:
            reward -= 1
        if agent.crossover:
            reward += 1
        return (
         (
          agent.id, reward), (agent.id, killed_id))

    def get_prey_reward(self, agent):
        reward = 0
        if agent.dead:
            reward -= 1
        if agent.crossover:
            reward += 1
        return ((agent.id, reward), (agent.id, None))

    def get_reward(self, agent):
        if agent.predator:
            return self.get_predator_reward(agent)
        else:
            return self.get_prey_reward(agent)

    def remove_dead_agents(self):
        killed = []
        for agent in self.agents.values():
            if agent.health <= 0:
                x, y = agent.pos
                self.map[x][y] = 0
                if agent.predator:
                    del self.predators[agent.id]
                    self.predator_num -= 1
                else:
                    del self.preys[agent.id]
                    self.prey_num -= 1
                killed.append(agent.id)
            elif agent.id in self.killed:
                killed.append(agent.id)
                del self.preys[agent.id]
                self.prey_num -= 1
                x, y = agent.pos
                self.map[x][y] = 0
            else:
                agent.age += 1
                agent.crossover = False

        self.killed = []
        return killed

    def _get_obs(self, agent):
        x, y = agent.pos
        obs = np.zeros((4 + self.agent_emb_dim, self.vision_width, self.vision_height))
        left = np.maximum(x - self.vision_width // 2, 0)
        right = np.minimum(x - self.vision_width // 2 + self.vision_width, self.w)
        top = np.maximum(y - self.vision_height // 2, 0)
        bottom = np.minimum(y - self.vision_height // 2 + self.vision_height, self.h)
        local_map = self.map[left:right, top:bottom]
        obs[4:, self.vision_width // 2, self.vision_height // 2] = self.agent_embeddings[agent.id]
        object_indice = np.where(local_map != 0)
        local_width, local_height = local_map.shape
        x_offset = self.vision_width - local_width
        y_offset = self.vision_height - local_height
        x_offset = 0
        y_offset = 0
        if x - self.vision_width // 2 < 0:
            x_offset = -(x - self.vision_width // 2)
            obs[0, :x_offset] = self.property[(-1)][1][0]
            obs[1, :x_offset] = self.property[(-1)][1][1]
            obs[2, :x_offset] = self.property[(-1)][1][2]
        if y - self.vision_height // 2 < 0:
            y_offset = -(y - self.vision_height // 2)
            obs[0, :, :y_offset] = self.property[(-1)][1][0]
            obs[1, :, :y_offset] = self.property[(-1)][1][1]
            obs[2, :, :y_offset] = self.property[(-1)][1][2]
        if x - self.vision_width // 2 + self.vision_width > self.w:
            right_offset = x - self.vision_width // 2 + self.vision_width - self.w
            obs[0, -right_offset:] = self.property[(-1)][1][0]
            obs[1, -right_offset:] = self.property[(-1)][1][1]
            obs[2, -right_offset:] = self.property[(-1)][1][2]
        if y - self.vision_height // 2 + self.vision_height > self.h:
            bottom_offset = y - self.vision_height // 2 + self.vision_height - self.h
            obs[0, :, -bottom_offset:] = self.property[(-1)][1][0]
            obs[1, :, -bottom_offset:] = self.property[(-1)][1][1]
            obs[2, :, -bottom_offset:] = self.property[(-1)][1][2]
        for object_x, object_y in zip(object_indice[0], object_indice[1]):
            if local_map[(object_x, object_y)] > 0:
                other_agent = self.agents[local_map[(object_x, object_y)]]
                agent_id = other_agent.id
                object_x += x_offset
                object_y += y_offset
                obs[(0, object_x, object_y)] = other_agent.property[1][0]
                obs[(1, object_x, object_y)] = other_agent.property[1][1]
                obs[(2, object_x, object_y)] = other_agent.property[1][2]
                obs[(3, object_x, object_y)] = other_agent.health
            elif local_map[(object_x, object_y)] == -1:
                object_x += x_offset
                object_y += y_offset
                obs[(0, object_x, object_y)] = self.property[(-1)][1][0]
                obs[(1, object_x, object_y)] = self.property[(-1)][1][1]
                obs[(2, object_x, object_y)] = self.property[(-1)][1][2]

        if self.obs_type == 'dense':
            return (agent.id, obs[:4].reshape(-1))
        else:
            return (
             agent.id, obs)

    def _get_all(self, agent):
        x, y = agent.pos
        obs = np.zeros((4 + self.agent_emb_dim, self.vision_width, self.vision_height))
        left = np.maximum(x - self.vision_width // 2, 0)
        right = np.minimum(x - self.vision_width // 2 + self.vision_width, self.w)
        top = np.maximum(y - self.vision_height // 2, 0)
        bottom = np.minimum(y - self.vision_height // 2 + self.vision_height, self.h)
        local_map = self.map[left:right, top:bottom]
        obs[4:, self.vision_width // 2, self.vision_height // 2] = self.agent_embeddings[agent.id]
        object_indice = np.where(local_map != 0)
        x_offset = 0
        y_offset = 0
        if x - self.vision_width // 2 < 0:
            x_offset = -(x - self.vision_width // 2)
            obs[0, :x_offset] = self.property[(-1)][1][0]
            obs[1, :x_offset] = self.property[(-1)][1][1]
            obs[2, :x_offset] = self.property[(-1)][1][2]
        if y - self.vision_height // 2 < 0:
            y_offset = -(y - self.vision_height // 2)
            obs[0, :, :y_offset] = self.property[(-1)][1][0]
            obs[1, :, :y_offset] = self.property[(-1)][1][1]
            obs[2, :, :y_offset] = self.property[(-1)][1][2]
        if x - self.vision_width // 2 + self.vision_width > self.w:
            right_offset = x - self.vision_width // 2 + self.vision_width - self.w
            obs[0, -right_offset:] = self.property[(-1)][1][0]
            obs[1, -right_offset:] = self.property[(-1)][1][1]
            obs[2, -right_offset:] = self.property[(-1)][1][2]
        if y - self.vision_height // 2 + self.vision_height > self.h:
            bottom_offset = y - self.vision_height // 2 + self.vision_height - self.h
            obs[0, :, -bottom_offset:] = self.property[(-1)][1][0]
            obs[1, :, -bottom_offset:] = self.property[(-1)][1][1]
            obs[2, :, -bottom_offset:] = self.property[(-1)][1][2]
        for object_x, object_y in zip(object_indice[0], object_indice[1]):
            if local_map[(object_x, object_y)] > 0:
                other_agent = self.agents[local_map[(object_x, object_y)]]
                agent_id = other_agent.id
                object_x += x_offset
                object_y += y_offset
                obs[(0, object_x, object_y)] = other_agent.property[1][0]
                obs[(1, object_x, object_y)] = other_agent.property[1][1]
                obs[(2, object_x, object_y)] = other_agent.property[1][2]
                obs[(3, object_x, object_y)] = other_agent.health
            elif local_map[(object_x, object_y)] == -1:
                object_x += x_offset
                object_y += y_offset
                obs[(0, object_x, object_y)] = self.property[(-1)][1][0]
                obs[(1, object_x, object_y)] = self.property[(-1)][1][1]
                obs[(2, object_x, object_y)] = self.property[(-1)][1][2]

        rewards, killed = self.get_reward(agent)
        if self.obs_type == 'dense':
            return ((agent.id, obs[:4].reshape(-1)), rewards, killed)
        else:
            return (
             (
              agent.id, obs), rewards, killed)

    def remove_dead_agent_emb(self, dead_list):
        for id in dead_list:
            del self.agent_embeddings[id]

    def render(self, only_view=False):
        if self.cpu_cores is None:
            cores = mp.cpu_count()
        else:
            cores = self.cpu_cores
        if self.args.multiprocessing:
            pool = mp.Pool(processes=cores)
            if only_view:
                obs = pool.map(self._get_obs, self.agents.values())
            else:
                obs = pool.map(self._get_all, self.agents.values())
            pool.close()
            pool.join()
        else:
            obs = []
            if only_view:
                for agent in self.agents.values():
                    obs.append(self._get_obs(agent))

            else:
                for agent in self.agents.values():
                    obs.append(self._get_all(agent))

        return obs

    def reset(self):
        self.__init__(self.args)
        self.make_world(wall_prob=(self.args.wall_prob), wall_seed=(self.args.wall_seed), food_prob=(self.args.food_prob))
        return self.render

    def step(self, actions):
        self.timestep += 1
        self.take_actions(actions)
        rewards = {}
        obs, rewards, killed = zip(*self.render())
        self.killed = list(dict(killed).values())
        for id, killed in killed:
            if killed is not None:
                self.increase_health(self.agents[id])

        return (
         obs, dict(rewards))


import numpy as np
from PIL import Image, ImageEnhance
import functools as tools
import ctypes
from os import listdir
import matplotlib.pyplot as plt
from os.path import isfile, join

def load_image_array(fileName):
	image = Image.open(fileName)
	arr = np.asarray(image).astype(np.int16)
	return arr

def get_images_delta(first_array, second_array):
	return np.subtract(first_array, second_array)

def calculate_noise_distribution(array_collection):
	concat = np.concatenate(array_collection).ravel().tolist()
	return concat

def value_cmp(a, b):
    if a[1] > b[1]:
        return -1
    elif a[1] == b[1]:
        if a[0] > b[0]:
            return 1
        else:
            return -1
    else:
        return 1

def add_ocurrency_to_dictionary(element, dictionary):
	if(element in dictionary):
		dictionary[element] += 1
	else:
		dictionary[element]= 1
	return element

def dictionary_to_arrays(ocurrencies_dict):
	keys = ocurrencies_dict.keys()
	elements = []
	for key in keys:
		elements.append((key, ocurrencies_dict[key]))
	sorted_elements = sorted(elements, key=lambda x: x[0])
	keys = []
	values = []
	for element in sorted_elements:
		keys.append(element[0])
		values.append(element[1])
	return keys, values

def get_plotable_arrays(ocurrencies_list):
	ocurrencies_dictionary = {}
	fillDictionary = lambda x: add_ocurrency_to_dictionary(x,ocurrencies_dictionary)
	mappingFunction = np.vectorize(fillDictionary)
	mappingFunction(ocurrencies_list)
	return dictionary_to_arrays(ocurrencies_dictionary)

def main(mypath):
	files = [f for f in listdir(mypath) if isfile(join(mypath, f))]
	last_file = None
	deltas_list = []
	for file in files:
		if(not last_file):
			last_file=file
			continue
		last_image = load_image_array(mypath+'/'+file)
		current_image = load_image_array(mypath+'/'+last_file)
		deltas = get_images_delta(last_image, current_image)
		deltas_list.append(deltas)
		last_file=file
	ocurrencies = calculate_noise_distribution(deltas_list[:])
	print(len(deltas_list), "Fotos se procesaron")
	keys, values = get_plotable_arrays(ocurrencies)
	correctable_pixels = tools.reduce(lambda x,y: x+y, keys[7:])
	print(correctable_pixels)
	hist, bins = np.histogram(values, keys)
	plt.hist(hist, bins)
	plt.yscale('log')
	plt.show()

main('./canal_1')


import os

from time import strftime

import h5py
import numpy as np
import pandas as pd

SONYC_PATH = '/beegfs/work/sonyc/'
OPEN_L3 = 'features/openl3/2017/'

node_paths = os.listdir(SONYC_PATH + OPEN_L3)

rainy_df = pd.read_csv('data/audio-paths-rained.csv')
nonrainy_df = pd.read_csv('data/audio-paths-nonrained.csv')

rainy_nodes = rainy_df['path'].map(lambda _: _.split('/')[2])
nonrainy_nodes = nonrainy_df['path'].map(lambda _: _.split('/')[2])

node_set = set(rainy_nodes) & set(nonrainy_nodes)

for i, node_path in enumerate(node_paths):
    node = node_path.split('_')[0]

    print(strftime(f'[%T] On node {node}'))

    rainy_matches = rainy_df[rainy_nodes == node]['node_timestamp'].values
    nonrainy_matches = nonrainy_df[nonrainy_nodes == node]['node_timestamp'].values

    if not rainy_matches.any() or not nonrainy_matches.any():
        print(strftime('[%T] No matches found for this node'))
        continue

    feature_path = SONYC_PATH + OPEN_L3 + node_path
    with h5py.File(feature_path, 'r') as feature_file:
        print(strftime('[%T] Extracting feature'))

        timestamps = feature_file['openl3']['timestamp']

        xy_rainy, feature_rainy_ind, rainy_ind = np.intersect1d(
            timestamps,
            rainy_matches,
            return_indices=True
        )
        xy_nonrainy, feature_nonrainy_ind, nonrainy_ind = np.intersect1d(
            timestamps,
            nonrainy_matches,
            return_indices=True
        )

        if not any(feature_rainy_ind) or not any(feature_nonrainy_ind):
            print(strftime('[%Y] No overlap found for this node'))
            continue

        with h5py.File(os.getcwd() + f'/nodes/{node}-features.h5', 'w') as write_file:
            print(strftime(f'[%T] Creating {node}-features.h5 file'))

            write_file.create_dataset(
                'rainy',
                data=feature_file['openl3'][list(np.sort(feature_rainy_ind))]
            )
            write_file.create_dataset(
                'nonrainy',
                data=feature_file['openl3'][list(np.sort(feature_nonrainy_ind))]
            )


# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

#!/usr/bin/env python2.7
"""Export inception model given existing training checkpoints.
The model is exported as SavedModel with proper signatures that can be loaded by
standard tensorflow_model_server.
"""

from __future__ import print_function

import os.path

import tensorflow as tf
import argparse

from core.yolov3 import YOLOV3
from tensorflow.python.compiler.tensorrt import trt_convert as trt
from tensorflow.tools.graph_transforms import TransformGraph
from tensorflow.python import ops
import numpy as np

IMAGE_SIZE = 416

output_node_names = ["encoded_image_tensor", "pred_sbbox/concat_2", "pred_mbbox/concat_2", "pred_lbbox/concat_2"]
transforms = ['remove_nodes(op=Identity)',
              'merge_duplicate_nodes',
              'strip_unused_nodes',
              'fold_constants(ignore_errors=true)',
              'fold_batch_norms',
              'quantize_weights']  # this reduces the size, but there is no speed up , actaully slows down, see below

# transforms = ['add_default_attributes',
# 'strip_unused_nodes',
# 'remove_nodes(op=Identity, op=CheckNumerics)',
# 'fold_constants(ignore_errors=true)',
# 'fold_batch_norms',
# 'fold_old_batch_norms',
# 'quantize_weights',
# 'quantize_nodes',
# 'strip_unused_nodes',
# 'sort_by_execution_order']

parser = argparse.ArgumentParser(description='Generate a saved model.')

parser.add_argument("--model_version", default="1", help="Version number of the model.", type=str)
parser.add_argument("--output_dir", default="model/yolov3", help="export model directory", type=str)
parser.add_argument("--output_trt_dir", default="model/yolov3/trt", help="export model directory", type=str)
parser.add_argument("--ckpt_dir", default="checkpoint", help="Directory where to read training checkpoints.", type=str)
parser.add_argument("--input_tensor", default="encoded_image_tensor:0", help="input tensor", type=str)
parser.add_argument("--sbbox_tensor", default="pred_sbbox/concat_2:0", help="pred_sbbox", type=str)
parser.add_argument("--mbbox_tensor", default="pred_mbbox/concat_2:0", help="pred_mbbox", type=str)
parser.add_argument("--lbbox_tensor", default="pred_lbbox/concat_2:0", help="pred_lbbox", type=str)
parser.add_argument("--class_num", default="80", help="class num", type=int)

parser.add_argument("--freeze_graph_dir", default="coco/yolov3_helmet.pb",
                    help="Directory where to save yolov3_helmet.pb.", type=str)
parser.add_argument("--optimizer_graph_dir", default="model/optimized_yolov3_helmet.pb",
                    help="Directory where to save optimized_yolov3_helmet.pb.", type=str)

args = parser.parse_args()

def get_graph_def_from_file(graph_filepath):
    tf.reset_default_graph()
    with ops.Graph().as_default():
        with tf.gfile.GFile(graph_filepath, 'rb') as f:
            graph_def = tf.GraphDef()
            graph_def.ParseFromString(f.read())
            return graph_def

def optimize_graph(model_dir, graph_filename, transforms, output_names, outname='optimized_model.pb'):
    input_names = ['input_image',] # change this as per how you have saved the model
    graph_def = get_graph_def_from_file(os.path.join(model_dir, graph_filename))
    optimized_graph_def = TransformGraph(
    graph_def,
    input_names,
    output_names,
    transforms)
    tf.train.write_graph(optimized_graph_def,
    logdir=model_dir,
    as_text=False,
    name=outname)
    print('Graph optimized!')

def export() -> object:

    serialized_tf_example = tf.placeholder(tf.string, shape=[None, 1], name='encoded_image_tensor')
    # batch_size = tf.shape(serialized_tf_example)[0]
    # serialized_tf_example = tf.reshape(serialized_tf_example, shape=[batch_size])
    # with tf.control_dependencies([tf.print(serialized_tf_example)]):
    images = tf.map_fn(preprocess_image, serialized_tf_example, tf.float32)

    model = YOLOV3(images, trainable=False)
    print(model.conv_sbbox, model.conv_mbbox, model.conv_lbbox)

    sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
    saver = tf.train.Saver()
    ckpt = tf.train.get_checkpoint_state(args.ckpt_dir)
    saver.restore(sess, ckpt.model_checkpoint_path)

    converted_graph_def = tf.graph_util.convert_variables_to_constants(sess,
                                                                       input_graph_def=sess.graph.as_graph_def(),
                                                                       output_node_names=output_node_names)

    with tf.gfile.GFile(args.freeze_graph_dir, "wb") as f:
        f.write(converted_graph_def.SerializeToString())

    optimize_graph('', args.freeze_graph_dir,
                   transforms, output_node_names, outname=args.optimizer_graph_dir)

    graph_def = get_graph_def_from_file(args.optimizer_graph_dir)
    # tf.reset_default_graph()
    # graph = tf.Graph()
    # sess = tf.Session()
    # # Create a TensorRT inference graph from a SavedModel:
    # with graph.as_default():
    #     with tf.Session() as sess:
    #         trt_graph = trt.create_inference_graph(
    #             input_graph_def=graph_def,
    #             outputs=output_node_names,
    #             max_batch_size=16,
    #             max_workspace_size_bytes=4000000000,
    #             is_dynamic_op=True,
    #             # precision_mode="FP16")
    #             # precision_mode=’FP32')
    #             precision_mode="INT8")
    #
    # trt_graph = trt.TrtGraphConverter.convert()

        # output_node = tf.import_graph_def(trt_graph, return_elements=output_node_names)
        # # sess.run(output_node)
        # tf.saved_model.simple_save(sess,
        # 'model/optimized_yolov3_helmet_tensorTR.pb',
        # inputs = {‘input_image’: graph.get_tensor_by_name(‘{}: 0’.format(node.name))
        # for node in graph.as_graph_def().node if node.op ==’Placeholder’},
        # outputs={t:graph.get_tensor_by_name(‘import / ’+t) for t in outputs}
        #
        # )

    with tf.Graph().as_default():

        tf.import_graph_def(
            graph_def,
            input_map=None,
            return_elements=None,
            name=""
        )

    with tf.Session() as sess:

        # Export inference model.
        output_path = os.path.join(
        tf.compat.as_bytes(args.output_dir),
        tf.compat.as_bytes(str(args.model_version)))
        print('Exporting trained model to', output_path)
        builder = tf.saved_model.builder.SavedModelBuilder(output_path)

        input_tensor = tf.get_default_graph().get_tensor_by_name(args.input_tensor)
        sbbox_tensor = tf.get_default_graph().get_tensor_by_name(args.sbbox_tensor)
        mbbox_tensor = tf.get_default_graph().get_tensor_by_name(args.mbbox_tensor)
        lbbox_tensor = tf.get_default_graph().get_tensor_by_name(args.lbbox_tensor)
        class_num = args.class_num

        batch_size = tf.shape(input_tensor)[0]
        sbbox_tensor = tf.reshape(tf.convert_to_tensor(sbbox_tensor), shape=[batch_size, -1, 5 + class_num])
        mbbox_tensor = tf.reshape(tf.convert_to_tensor(mbbox_tensor), shape=[batch_size, -1, 5 + class_num])
        lbbox_tensor = tf.reshape(tf.convert_to_tensor(lbbox_tensor), shape=[batch_size, -1, 5 + class_num])
        output_tensor = tf.concat([sbbox_tensor, mbbox_tensor, lbbox_tensor], 1)
        # top_100 = tf.nn.top_k(output_tensor[:, 4], 100)
        # output_tensor = tf.gather(output_tensor, top_100.indices)
        num_tensor = tf.cast(tf.tile([[100]], multiples=[batch_size, 1]), dtype=tf.int32)

        classes_tensor = tf.argmax(output_tensor[:, :, 5:], axis=2)
        scores_tensor = output_tensor[:, :, 4]
        raw_boxs_tensor = output_tensor[:, :, 0:4] / IMAGE_SIZE
        print(raw_boxs_tensor.shape)
        boxs_tensor_com = raw_boxs_tensor
        print(boxs_tensor_com.shape)

        boxs_tensor_minx = raw_boxs_tensor[:, :, 0] - raw_boxs_tensor[:, :, 2] * 0.5
        boxs_tensor_miny = raw_boxs_tensor[:, :, 1] - raw_boxs_tensor[:, :, 3] * 0.5
        boxs_tensor_maxx = raw_boxs_tensor[:, :, 0] + raw_boxs_tensor[:, :, 2] * 0.5
        boxs_tensor_maxy = raw_boxs_tensor[:, :, 1] + raw_boxs_tensor[:, :, 3] * 0.5
        boxs_tensor_minx = tf.expand_dims(boxs_tensor_minx, 2)
        boxs_tensor_miny = tf.expand_dims(boxs_tensor_miny, 2)
        boxs_tensor_maxx = tf.expand_dims(boxs_tensor_maxx, 2)
        boxs_tensor_maxy = tf.expand_dims(boxs_tensor_maxy, 2)

        boxs_tensor_com = tf.concat([boxs_tensor_miny, boxs_tensor_minx, boxs_tensor_maxy, boxs_tensor_maxx], 2)
        scores_tensor_map = tf.expand_dims(scores_tensor, 2)
        classes_tensor_map = tf.expand_dims(classes_tensor, 2)
        classes_tensor_map = tf.cast(classes_tensor_map, dtype=tf.float32)
        nms_tensor_map = tf.concat([boxs_tensor_com, scores_tensor_map, classes_tensor_map], 2)
        tensor_result = tf.map_fn(preprocess_nms, nms_tensor_map, tf.float32, name="map2")

        # with tf.control_dependencies([tf.print(tensor_result[0, :, 5])]):
        #     tmpt = tensor_result[:, :, 5]
        scores_tensor_info = tf.saved_model.utils.build_tensor_info(tensor_result[:, :, 5])
        classes_tensor_info = tf.saved_model.utils.build_tensor_info(tf.cast(tensor_result[:, :, 4], dtype=tf.int64))
        boxes_tensor_info = tf.saved_model.utils.build_tensor_info(tensor_result[:, :, 0:4])
        raw_boxes_tensor_info = tf.saved_model.utils.build_tensor_info(raw_boxs_tensor)
        num_tensor_info = tf.saved_model.utils.build_tensor_info(num_tensor)

        print(scores_tensor_info)
        print(classes_tensor_info)
        print(boxes_tensor_info)
        print(num_tensor_info)

        # scores_tensor_info = tf.saved_model.utils.build_tensor_info(tf.expand_dims(scores_tensor, 0))
        # classes_tensor_info = tf.saved_model.utils.build_tensor_info(tf.expand_dims(classes_tensor, 0))
        # boxes_tensor_info = tf.saved_model.utils.build_tensor_info(tf.expand_dims(boxs_tensor_com, 0))
        # raw_boxes_tensor_info = tf.saved_model.utils.build_tensor_info(tf.expand_dims(raw_boxs_tensor, 0))
        # num_tensor_info = tf.saved_model.utils.build_tensor_info(tf.expand_dims(num_tensor, 0))

        inputs_tensor_info = tf.saved_model.utils.build_tensor_info(
        input_tensor)

        tensor_info_inputs = {
            'inputs': inputs_tensor_info
        }
        print(scores_tensor_info, inputs_tensor_info, classes_tensor_info, boxes_tensor_info, num_tensor_info)

        prediction_signature = (
            tf.saved_model.signature_def_utils.build_signature_def(
            inputs=tensor_info_inputs,
            # outputs=tensor_info_outputs,
            outputs={
            'detection_scores': scores_tensor_info,
            'detection_classes': classes_tensor_info,
            'detection_boxes': boxes_tensor_info,
            # 'raw_boxes': raw_boxes_tensor_info,
            'num_detections': num_tensor_info,
            },
            method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME
            ))

        builder.add_meta_graph_and_variables(
            sess,
            [tf.saved_model.tag_constants.SERVING],
            signature_def_map={
            tf.saved_model.signature_constants
            .DEFAULT_SERVING_SIGNATURE_DEF_KEY:
            prediction_signature,
            },
        )

        builder.save()
        print('Successfully exported model to %s' % args.output_dir)

        print("start tensorrt optimization")
        converter = trt.TrtGraphConverter(input_saved_model_dir="/home/zyc/tensorflow-yolov3-master/model/yolov3/1",
                                          max_batch_size=16,
                                          max_workspace_size_bytes=4000000000,
                                          is_dynamic_op=True,
                                          precision_mode="FP16")
        graph_def = converter.convert()

# filename = 'bus.jpg'
# with open(filename, 'rb') as f:
#     data = f.read()
#
# calibrated_graph_def = converter.calibrate(
#     fetch_names=['softmax_tensor'],
#     num_runs=1,
#     feed_dict_fn=lambda: {'inputs:0': np.array([data])}
# )
        converter.save(args.output_trt_dir)

def preprocess_image(image_buffer):
  """Preprocess JPEG encoded bytes to 3D float Tensor."""

  # Decode the string as an RGB JPEG.
  # Note that the resulting image contains an unknown height and width
  # that is set dynamically by decode_jpeg. In other words, the height
  # and width of image is unknown at compile-time.
  # image = tf.io.decode_base64(image_buffer)
  # tf.image.
  # with tf.control_dependencies([tf.print(image_buffer)]):
  image = tf.image.decode_jpeg(image_buffer[0], channels=3)

  image.set_shape((None, None, 3))
  image_paded = tf.image.resize(image, [IMAGE_SIZE, IMAGE_SIZE], method=0)
  return image_paded/255

def preprocess_nms(nms_tensor_map):
  """Preprocess nms float Tensor."""
  boxs_tensor_box = nms_tensor_map[:, 0:4]
  scores_tensor = nms_tensor_map[:, 4]
  classes_tensor = nms_tensor_map[:, 5]
  boxs_tensor_indices = tf.image.non_max_suppression(boxs_tensor_box, scores_tensor, 100, 0.5)
  boxs_tensor_indices = tf.reshape(boxs_tensor_indices, shape=[100])

  boxs_tensor_com = tf.gather(boxs_tensor_box, boxs_tensor_indices)
  # boxs_tensor_com = tf.reshape(boxs_tensor_com, shape=[-1, 100, 4])
  classes_tensor = tf.gather(classes_tensor, boxs_tensor_indices)
  classes_tensor = tf.expand_dims(classes_tensor, 1)
  scores_tensor = tf.gather(scores_tensor, boxs_tensor_indices)
  scores_tensor = tf.expand_dims(scores_tensor, 1)
  result = tf.concat([boxs_tensor_com, classes_tensor, scores_tensor], 1)

  return result

def main(unused_argv=None):
  export()

if __name__ == '__main__':
  tf.app.run()


from __future__ import print_function
import sys
from subprocess import Popen, PIPE

import flask_login as login
import flask_admin as admin
from flask_admin import helpers, expose
from flask import redirect, url_for, request, render_template

from loginform import LoginForm
from atfform import JobForm
import stub as stub

class AdminIndexView(admin.AdminIndexView):

    def _stubs(self):
        self.nav = {
            "tasks" : stub.get_tasks(),
            "messages" : stub.get_messages_summary(),
            "alerts" : stub.get_alerts()
        }

        (cols, rows) = stub.get_adv_tables()
        (scols, srows, context) = stub.get_tables()

        self.tables = {
            "advtables" : { "columns" : cols, "rows" : rows },
            "table" : { "columns" : scols, "rows" : srows, "context" : context}
        }

        self.panelswells = {
            "accordion" : stub.get_accordion_items(),
            "tabitems" : stub.get_tab_items()
        }

    @expose('/')
    def index(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Dashboard"
        return render_template('sb-admin/pages/dashboard.html', admin_view=self)

    @expose('/atf', methods=('GET', 'POST'))
    def atf(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        form = JobForm(request.form)
        msg = ""
        if request.method == 'POST':
            command1 = str(form.get_username())
            # print ("cmd: " + command1, file=sys.stderr)
            cmd = [ command1 ]
            p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
            output, err = p.communicate()
            rc = p.returncode
            msg = str(output) + str(err)
            # print("out: " + msg, file=sys.stderr)
            # print("err: " + str(err), file=sys.stderr)
            # print("rc: " + str(rc), file=sys.stderr)

        msg = msg.decode('utf-8', 'ignore')

        self._stubs()
        self.header = "ATF"
        self._template_args['form'] = form
        return render_template('sb-admin/pages/atf.html', admin_view=self, form=form, msg=msg)

    @expose('/login/', methods=('GET', 'POST'))
    def login_view(self):
        # handle user login
        form = LoginForm(request.form)
        if helpers.validate_form_on_submit(form):
            user = form.get_user()
            login.login_user(user)

        if login.current_user.is_authenticated:
            return redirect(url_for('.index'))
        self._template_args['form'] = form
        return render_template('sb-admin/pages/login.html', form=form)

    @expose('/blank')
    def blank(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Blank"
        return render_template('sb-admin/pages/blank.html', admin_view=self)

    @expose('/flot')
    def flot(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Flot Charts"
        return render_template('sb-admin/pages/flot.html', admin_view=self)

    @expose('/morris')
    def morris(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Morris Charts"
        return render_template('sb-admin/pages/morris.html', admin_view=self)

    @expose('/tables')
    def tables(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Tables"
        return render_template('sb-admin/pages/tables.html', admin_view=self)

    @expose('/forms')
    def forms(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Forms"
        return render_template('sb-admin/pages/forms.html', admin_view=self)

    @expose('/ui/panelswells')
    def panelswells(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Panels Wells"
        return render_template('sb-admin/pages/ui/panels-wells.html', admin_view=self)

    @expose('/ui/buttons')
    def buttons(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Buttons"
        return render_template('sb-admin/pages/ui/buttons.html', admin_view=self)

    @expose('/ui/notifications')
    def notifications(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Notifications"
        return render_template('sb-admin/pages/ui/notifications.html', admin_view=self)

    @expose('/ui/typography')
    def typography(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Typography"
        return render_template('sb-admin/pages/ui/typography.html', admin_view=self)

    @expose('/ui/icons')
    def icons(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Icons"
        return render_template('sb-admin/pages/ui/icons.html', admin_view=self)

    @expose('/ui/grid')
    def grid(self):
        if not login.current_user.is_authenticated:
            return redirect(url_for('.login_view'))

        self._stubs()
        self.header = "Grid"
        return render_template('sb-admin/pages/ui/grid.html', admin_view=self)

    @expose('/logout/')
    def logout_view(self):
        login.logout_user()
        return redirect(url_for('.index'))

class BlankView(admin.BaseView):
    @expose('/')
    def index(self):
        return render_template('sb-admin/pages/blank.html', admin_view=self)


#! /usr/bin/env python

"""
Tool to find messages in log files and filter them if they are already 'waived'.

@see: http://www.kiatoa.com/cgi-bin/fossils/logpro/home

Copyright (c) OIP 2016
"""

import argparse
import logging
import os
import re
import sys

import tabulate
from six import exec_

# Logging handler
#
logging.basicConfig(level=logging.DEBUG)

#
# argument parsing
#
parser = argparse.ArgumentParser()
parser.add_argument('input',
                    nargs='?',
                    help="log file")
args = parser.parse_args()

# code taken from Sphinx
def execfile_(filepath, _globals, open=open):
    from sphinx.util.osutil import fs_encoding
    # get config source -- 'b' is a no-op under 2.x, while 'U' is
    # ignored under 3.x (but 3.x compile() accepts \r\n newlines)
    with open(filepath, 'rbU') as f:
        source = f.read()

    # py26 accept only LF eol instead of CRLF
    if sys.version_info[:2] == (2, 6):
        source = source.replace(b'\r\n', b'\n')

    # compile to a code object, handle syntax errors
    filepath_enc = filepath.encode(fs_encoding)
    code = compile(source, filepath_enc, 'exec')
    exec_(code, _globals)

# founded: https://gist.github.com/zdavkeos/1098474
def walk_up(bottom):
    """
    mimic os.walk, but walk 'up'
    instead of down the directory tree
    """

    bottom = os.path.realpath(bottom)

    # get files in current dir
    try:
        names = os.listdir(bottom)
    except Exception as e:
        print(e)
        return

    dirs, nondirs = [], []
    for name in names:
        if os.path.isdir(os.path.join(bottom, name)):
            dirs.append(name)
        else:
            nondirs.append(name)

    yield bottom, dirs, nondirs

    new_path = os.path.realpath(os.path.join(bottom, '..'))

    # see if we are at the top
    if new_path == bottom:
        return

    for x in walk_up(new_path):
        yield x

if __name__ == '__main__':

    # open config file .log4u in the CWD or below
    # so: a .log4u file in '..' should be found
    config = {}
    filename = '.log4u'
    for root, subfolders, files in walk_up(os.getcwd()):
        if filename in files:
            config_file = os.path.join(root, filename)
            break  # first config file founded -> stop
    config['__file__'] = config_file
    execfile_(config_file, config)

    # open a log file
    # @todo: for the moment, only srr files of Synplify
    with open(args.input) as f:
        log = f.readlines()

    # add the scoreboard info to the config variables
    # config['WAIVERS'] = [w['count']=None for w in config['WAIVERS']]
    for w in config['WAIVERS']:
        w.update({'count': 0})  # number of triggers of this waiver
        w.update({'match': []})  # messages waived away

    log_modif = []  # modified log

    scoreboard = {'error': {'number': 0, 'waived': 0},
                  'warning': {'number': 0, 'waived': 0, 'non_waived': []},
                  'attention': {'number': 0, 'waived': 0}}

    # modify the waived file
    for line in log:
        if re.search('^@W:', line):  # only for warnings
            scoreboard['warning']['number'] += 1
            _trigger = None  # warning is waived away if True
            for waive in config['WAIVERS']:
                if waive['pattern'].search(line):
                    _trigger = True  # found a match, warning will be waived
                    log_modif.append(":LOG4U:" + line)
                    log_modif.append(":LOG4U: reason: {}\n".format(waive['reason']))
                    scoreboard['warning']['waived'] += 1
                    waive.update({'count': waive['count'] + 1})
                    waive['match'].append(line.strip())
                else:  # nothing found to waive away
                    log_modif.append(line)
            if _trigger is None:  # warning did not match any pattern
                scoreboard['warning']['non_waived'].append(line.strip())

        if re.search('^@E:', line):
            scoreboard['error']['number'] += 1

        if re.search('^@A:', line):
            scoreboard['attention']['number'] += 1
        else:
            log_modif.append(line)

    with open(args.input + ".log4u", 'w') as f:
        logging.info("Writing to file '{:s}'".format(f.name))
        f.writelines(log_modif)

    # RestructuredText Log file
    rst = ""
    _header = "Synthesis"  # @todo extract this out of the config file
    rst += "{:s}\n".format(_header)
    rst += "-" * len(_header) + "\n"
    rst += "\n"
    table_summary = [['', 'Total', 'Waived'],
                     ['Error', scoreboard['error']['number'], scoreboard['error']['waived']],
                     ['Warning', scoreboard['warning']['number'], scoreboard['warning']['waived']],
                     ['Attention', scoreboard['attention']['number'], scoreboard['attention']['waived']]]

    for l in tabulate.tabulate(table_summary,
                               tablefmt="grid",
                               headers='firstrow').splitlines():
        rst += "\t{}\n".format(l)  # add indentation

    rst += "\n"
    rst += "\n"

    for w in config['WAIVERS']:
        _title = "{}".format(w['pattern'].pattern.replace('\\', '\\\\'))
        rst += "{}\n".format(_title)
        rst += "^" * len(_title) + "\n"
        rst += "\n"
        rst += ":matched:\n"
        for m in w['match']:
            # @todo: this works only for Synplify messages
            # e.g.: @W: CL190 :"C:\projects\pilot\data\oip\camsync\rtl\camsync.vhd":75:4:75:5|Optimizing register bit prdata(2) to a constant 0
            rst += "\t #. {}\n".format(m.split('|')[1])
        rst += ":reason: {}\n".format(w['reason'])
        rst += "\n"
    rst += "\n"

    # report the not waived away message as a warning
    if len(scoreboard['warning']['non_waived']) > 0:
        rst += ".. WARNING:: Not waived away warnings:\n"
        rst += "\n"
        for nw in scoreboard['warning']['non_waived']:
            rst += "\t#. {}\n".format(nw)
    rst += "\n"

    # bring the RestructuredText to the stdout
    print(rst)


#coding=utf-8
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime
import matplotlib.dates as mdates
from matplotlib.dates import AutoDateLocator
import os
import sys
import os.path

# 将同一类型的图像汇总
base_path=r'E:\HW\练习数据\初赛文档\练习数据\csv'
end_date='2016-01-31'
start_date='2015-01-01'
flavors=['flavor1','flavor2','flavor3','flavor4','flavor5','flavor6','flavor7','flavor8','flavor9','flavor10','flavor11','flavor12','flavor13','flavor14','flavor15']
fileList=[]
date_l=[datetime.strftime(x,'%Y-%m-%d') for x in list(pd.date_range(start=start_date, end=end_date))]

def list_file(path):
    count=0
    for filename in os.listdir(path):
        if os.path.splitext(filename)[1] == '.csv':
            count = count+1
            print(filename)
            fileList.append(filename)
            #fp = open(dirname+os.sep+filename,'r')
            #print(len(fp.readlines())-1)
            #fp.close()
    print('total csv file: ',count)

def flavor_month(flavor_type):
    data_ori = pd.Series()
    for file in fileList:
        df = pd.read_csv(base_path+'\\'+file, header=None, names=['type', 'day'], sep='\t', usecols=[1, 2])
        if(flavor_type in df['type'].values):
            a = df.groupby(by='type').get_group(flavor_type)['day'].value_counts().sort_index()
            data_ori=data_ori.append(a)

    date_time = list(data_ori.index)
    value_l = []
    for ind in range(0, len(date_l)):
        if date_l[ind] in date_time:
            val_tmp=date_l[ind]
            tmp = data_ori[val_tmp]
        else:
            tmp = 0
        value_l.append(tmp)
    plt.figure(figsize=(16.39, 9.35))
    data_time_translation = [datetime.strptime(d, '%Y-%m-%d').date() for d in date_l]
    plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))  # 显示时间坐标的格式
    # autodates = AutoDateLocator()  # 时间间隔自动选取
    plt.xticks(pd.date_range(start_date, end_date, freq='M'))  # 时间间隔
    df_tmp=pd.DataFrame(data=value_l,index=data_time_translation)
    df_tmp.to_csv(base_path+r'\全部2'+'\\'+flavor_type + '.csv')

    plt.plot(data_time_translation, value_l, 'b', lw=2.5)
    plt.gcf().autofmt_xdate()  # 自动旋转日期标记
    plt.grid(True)
    plt.axis("tight")
    plt.xlabel('Time')
    plt.ylabel('count')
    plt.title('flavor')
    # plt.gca().xaxis.set_major_locator(autodates)
    plt.legend(loc=0)
    # plt.show()
    plt.savefig(base_path+r'\全部2'+'\\'+flavor_type + '.png')
    plt.close('all')

if __name__ == '__main__':
    list_file(u'E:\HW\练习数据\初赛文档\练习数据\csv')
    for flavor in flavors:
        flavor_month(flavor)

    # plt.show()


import torch
from torch import nn
import torch.nn.functional as F
import numpy as np

class LanguageModel(nn.Module):

    def __init__(self, vocab_size: int, embed_size: int, hidden_size: int, num_layers: int, dropout: float, weight_tying: bool) -> None:
        super(LanguageModel, self).__init__()
        self.dropout = dropout
        self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embed_size)
        self.rnn = nn.GRU(
            input_size=embed_size,
            hidden_size=hidden_size,
            num_layers=num_layers,
            bidirectional=False,
            dropout=dropout if num_layers > 1 else 0,
            batch_first=True
        )
        self.output_projection = nn.Sequential(
            nn.Linear(hidden_size, hidden_size),
            nn.Tanh(),
            nn.Linear(hidden_size, embed_size)
        )
        self.generator = nn.Linear(embed_size, vocab_size)
        if weight_tying:
            self.generator.weight = self.embedding.weight

    def load_pretrained_embeddings(self, **kwargs) -> None:
        assert ('path' in kwargs) ^ ('embedding' in kwargs)
        if 'path' in kwargs:
            embedding = np.load(kwargs['path'])
        else:
            embedding = kwargs['embedding']
        self.embedding.weight.data.copy_(torch.tensor(embedding))

    def forward(self, sentence: torch.Tensor) -> torch.Tensor:
        '''
        :param sentence: torch.LongTensor (batch_size, seq_len)
        :return logit: torch.FloatTensor (batch_size, seq_len, vocab_size)
        '''

        sentence = self.embedding(sentence)
        sentence = F.dropout(sentence, p=self.dropout, training=self.training)
        hidden, _ = self.rnn(sentence)
        output = self.output_projection(hidden)
        logit = self.generator(output)
        return logit


import uuid

node_type = ("SNode", "DNode")

class LinkList:
    """
    object that is the head of a LinkedList
    """
    type = None
    head = None
    end = None

    def __init__(self, val=None):
        """
        initializes the  LinkList object contains one node representing the head.

        :param val: value for the first node to be set. if value other than a "Node" object is given
        then a new single linked node will be initialized with the given value.
        """
        if val.__class__.__name__ in node_type:
            self.type = type(val)
            self.head = val
        else:
            n = SNode(val, None)
            self.type = type(n)
            self.head = n

    def append(self, val):
        """
        this will append a new node to the end of the chain.
        :param val: value/Node to append to the end
        :return: Self
        """
        n = self.head
        new = None
        while n.next is not None:
            n = n.next

        if val.__class__.__name__ in node_type:
            new = n.attach(val)
        else:
            new = n.attach(self.type(val))
        if self.type is DNode:
            self.end = new

        return self

    def is_valid(self):
        """
        check if the Linkedlist is valid,
        If a LinkedList head doesn't point to a Node than the LinkList object is considered invalid
        :return: bool
        """
        if self.head is None:
            return False
        return True

    def pop(self):
        """
        remove the head node and progress the prama head to the next in the chain.
        :return: Node
        """
        if self.head is None:
            print("Error: linked list doesn't have any node in the chain.")
            return None

        old, self.head = self.head.detach()
        return old

    def remove(self, idx=0):
        """
        remove a node at a given index.
        :param idx: index of the node to delete.
        :return: Node that was deleted else None.
        """
        if idx == 0:
            return self.pop()

        i = 0
        n = self.head
        try:
            while i < idx - 1:
                i += 1
                n = n.next
        except AttributeError:
            print("Method remove(): Index range is beyond the scope of the linked list.")
            return None

        old, n.next = n.next.detach()
        return old

    def insert(self, val=None, idx=0):
        if idx == 0:
            if val.__class__.__name__ in node_type:
                if isinstance(type(val), self.head):
                    val.next = self.head
                    self.head = val
                    return self
                else:
                    print("Error: nodes types must match.")
                    return None
            else:
                self.head = self.type(val, self.head)
                return self

        i = 0
        n = self.head
        while i < idx - 1:
            i += 1
            n = n.next

        n.next = n.next.insert(val)

        return self

    def reverse(self):
        """
        reverse a linked list.
        :return: Self
        """
        if self.type is DNode:
            self.end = self.head

        n = self.head
        p = None
        while n:
            p, n = n.reverse(p)

        self.head = p

        return self

    # TODO: finish logic for function.
    def convert(self, obj=None):
        """
        converts the linked type based on given Node type
        :param obj: Node type to convert existing linked list to
        :return: Self
        """

        if not self.is_valid() or not obj:
            print("Error: converting, linked list must either be valid or `obj` must not be None")
            return None

        pass

    def __str__(self):
        return str(self.head)

class SNode:
    value = None
    next = None

    def __init__(self, val=None, next=None):
        self.value = val
        self.next = next

    def attach(self, node):
        if not isinstance(node, type(self)):
            print("`node` must be a SNode (single linked node) and same node type")
            return None

        self.next = node

        return node

    def detach(self):
        node = self.next
        self.next = None
        return self, node

    def insert(self, val=None):
        if not isinstance(val, type(self)):
            val = SNode(val, self)
        else:
            val.next = self

        return val

    def reverse(self, node=None):
        n = self.next
        self.next = node
        node = self
        return node, n

    def __str__(self):
        return f"[{self.__class__.__name__}] value: {self.value}, next: {str(self.next)}"

class DNode:
    value = None
    next = None
    prev = None
    __name = None

    def __init__(self, val=None, next=None, prev=None):
        self.value = val
        self.next = next
        self.prev = prev
        self.__name = str(uuid.uuid4())[0:5]

    def attach(self, node):
        if not isinstance(node, type(self)):
            print("`node` must be a DNode (double linked node) and same node type")
            return None

        node.prev = self
        self.next = node
        return node

    def detach(self):
        node = self.next
        node.prev = self.prev

        self.prev = None
        self.next = None
        return self, node

    def insert(self, val=None):
        if not isinstance(val, type(self)):
            val = DNode(val, self, self.prev)
        else:
            val.next = self
            val.prev = self.prev

        return val

    def reverse(self, node=None):
        n = self.next
        self.next, self.prev = (self.prev, self.next)
        return self, n

    @property
    def name(self):
        return self.__name

    def __str__(self):
        return f"<{self.__class__.__name__ + '_' + self.__name}> = value: {self.value}, prev: [{None if not self.prev else self.__class__.__name__ + '_' + self.prev.name}], next: {str(self.next)}"

if __name__ == "__main__":
    phrase = ['rat', 'a', 'for', 'me', 'left', 'hat,', 'the', 'in', 'cat', 'The']
    ll = LinkList(SNode(phrase[0]))
    phrase = phrase[1::]

    for i in phrase:
        ll.append(i)

    # ll.insert(500, 1).insert(256, 0)
    # # print("------")
    #
    print("------")
    ll.reverse()
    # ll.remove(100)
    print(ll)


#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

import logging

logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.DEBUG)

import pelican_render_math

AUTHOR = "Klafyvel"
SITENAME = "KlafyBlog"
SITEURL = "https://klafyvel.me/blog/"

PATH = "content"

TIMEZONE = "Europe/Paris"

DEFAULT_LANG = "en"

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (
    ("My personnal page", "https://klafyvel.me/"),
    ("Pelican", "http://getpelican.com/"),
)

# Social widget
SOCIAL = (
    ("My Twitter account", "https://twitter.com/Klafyvel"),
    ("My Github account", "https://github.com/Klafyvel"),
)

DEFAULT_METADATA = {"status": "draft"}

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
# RELATIVE_URLS = True

PLUGIN_PATHS = ["plugins"]
PLUGINS = [pelican_render_math, "decent_images"]
RESPONSIVE_IMAGES = True


# exercise_B.1

import pandas as pd
import matplotlib.pyplot as plt
iris = pd.read_csv("iris.csv")
iris.describe().plot(kind = "area",fontsize=16, figsize = (15,8), table = True, colormap="Accent")
plt.xlabel('Statistics',)
plt.ylabel('Value')
plt.title("General Statistics of Iris Dataset")
plt.show()


import cv2 as cv
import numpy as np

img = cv.imread('C:/Users/ASUS/Desktop/Desktop_pics/pears/Pear1.jpg')
gray = cv.cvtColor(img, cv.COLOR_BGRA2GRAY)
edges = cv.Canny(gray, 75, 150)

lines = cv.HoughLinesP(edges, 1, np.pi/180, 30, maxLineGap=150)

for line in lines:
    x1, y1, x2, y2 = line[0]
    cv.line(img, (x1, y1), (x2, y2), (0, 255, 0), 3)

cv.imshow("Edges", edges)
cv.imshow("Image", img)
cv.waitKey(0)
cv.destroyAllWindows()


import os
import cv2
from threading import Thread
from utils.general import clean_str
import time

class LoadStreams:  # multiple IP or RTSP cameras
    def __init__(self, sources='streams.txt', img_size=640):
        self.mode = 'stream'
        self.img_size = img_size

        if os.path.isfile(sources):
            with open(sources, 'r') as f:
                sources = [x.strip() for x in f.read().strip().splitlines() if len(x.strip())]
        else:
            sources = [sources]

        n = len(sources)
        self.imgs = [None] * n
        self.sources = [clean_str(x) for x in sources]  # clean source names for later
        for i, s in enumerate(sources):
            # Start the thread to read frames from the video stream
            print('%g/%g: %s... ' % (i + 1, n, s), end='')
            cap = cv2.VideoCapture(eval(s) if s.isnumeric() else s)
            assert cap.isOpened(), 'Failed to open %s' % s
            w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
            h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
            self.fps = cap.get(cv2.CAP_PROP_FPS) % 100
            _, self.imgs[i] = cap.read()  # guarantee first frame
            thread = Thread(target=self.update, args=([i, cap]), daemon=True)
            print(' success (%gx%g at %.2f FPS).' % (w, h, self.fps))
            thread.start()
        print('')  # newline

    def update(self, index, cap):
        # Read next stream frame in a daemon thread
        n = 0
        while cap.isOpened():
            n += 1
            # _, self.imgs[index] = cap.read()
            cap.grab()
            if n == 4:  # read every 4th frame
                _, self.imgs[index] = cap.retrieve()
                n = 0
            time.sleep(0.01)  # wait time


#!/usr/bin/env python3
'''
Author: nilputs@nilput.com
see COPYRIGHTS file which is included in this project
'''

import sys
import re
import argparse
import time
from pprint import pprint as pprint
from types import SimpleNamespace as nms
sys.path.append('.')
import lrgen
from ids import *
from format_types import indent

DEBUG = 0

#not used
def generate_C_rule_data(table):
    nprint('''\
struct rule_s rules_data[] = {''')
    for i,rule in enumerate(table.index_to_rule):
        # print(i, rule.lhs, file=sys.stderr)
        # print(table.ctx.bnf.syms.get_symbol(rule.lhs).sym_num)
        if table.ctx.bnf.syms.get_symbol(rule.lhs) is None:
            print(rule,file=sys.stderr)
            print(table.ctx.bnf.syms.syms,file=sys.stderr)
            print(table.ctx.bnf.syms.syms['D'],file=sys.stderr)
            raise ValueError('none')

        nprint('{{ {rule_length:<{width}}, {rule_lhs:<{width}} }},'.format(rule_length = len(rule.seq),
                                                                           rule_lhs = table.ctx.bnf.syms.get_symbol(rule.lhs).sym_num,
                                                                           width=2))
    nprint('};\n')

def generate_C_states_data(table):
    nprint('''\
struct state_table_s states[{num_states}] = {{'''.format(num_states = len(table.rows)))

    nprint('\n')
    for i,row in enumerate(table.rows):
        nprint('''\
    {{''')
        for sym_idx,entry in enumerate(row):
            if entry is not None:
                entry_str = '{{ {},{} }},'.format(entry.action, entry.number) #when using .format(), {{ == {
            else:
                entry_str = '{ EA_REJ,EA_REJ }, '
            nprint('{:>{width}}'.format(entry_str, width=9))
        nprint('}},\n')

    nprint(sc('''
    };
    '''))

def generate_C_includes(table):
        nprint('#include "parse.h"\n')

def generate_represent_function(table,defs):
    nprint('const char *represent(int val){\n''')
    nprint('    switch (val){\n''')
    for k,v in defs.items():
        string = k if (not k.startswith('T_LITERAL')) else "'{}'".format(table.ctx.bnf.syms.syms[k].val)
        nprint('    case {}: return "{}";\n'.format(v,string))
    nprint('    default: return "INVALID_SYMBOL";\n')
    nprint('\n    }\n}\n')

def generate_C_parse_enum(table):
    enum = '\n#define DBG_PRINT' if DEBUG else ''
    enum+='''
enum PARSE_ENUM{{
    STACKLEN   = 100,
    SYMBOLSMAX = 50,
    {defs}
}};
'''
    ddict = {}
    for symbol in table.ctx.bnf.syms.syms.values():
            ddict[symbol.name] = symbol.sym_num

    enum_dict = {
            'EA_REDUCE':EA_REDUCE,
            'EA_ACC':   EA_ACC,
            'EA_SHIFT': EA_SHIFT,
            'EA_GOTO':  EA_GOTO,
            'EA_ACC'      :      EA_ACC ,
            'EA_REJ'      :      EA_REJ ,
            'EA_NONE'    :      EA_NONE ,
            }
    enum_dict.update(ddict)
    defs = '\n'
    for k,v in enum_dict.items():
        defs += '    {k} = {v},\n'.format(k=k, v=v)
    nprint(enum.format(defs = defs))
    if DEBUG:
        generate_represent_function(table,enum_dict)

def generate_C_data(table):
    generate_C_parse_enum(table)
    generate_C_includes(table)
    generate_C_states_data(table)
    generate_C_rule_data(table)

def genereate_basic_lexer(table):
    nprint('''
int next_input(){
    static char buff[128];
    static int idx = 0;
    int r;

    r = fgetc(stdin);
    while (1){
        switch(r){
            case ' ':
            case '\\n':
            case '\\t':
                r = fgetc(stdin);
                continue;
        }
        break;
    }

    if (r != EOF){
        buff[idx++] = r;
        buff[idx] = 0;
        printf("read '%s'\\n", buff);
    }
    ''')

    written = 0
    for i,symbol in enumerate(table.index_to_symbol):
        if not symbol.name.startswith('T_LITERAL'):
            continue
        c = '' if written == 0 else 'else '
        c += '''if (strncmp({arg0},{arg1},{slen}) == 0){{
            return {rv};
        }}
        '''

        c = c.format( arg0 = 'buff+idx-1',
                      arg1 = '"{}"'.format(symbol.val),
                      slen = '{}'.format(len(symbol.val)),
                      rv   =  '{}'.format(symbol.sym_num) )
        written += 1
        nprint(c)

    nprint('''
    if (r == EOF || r == ';'){
        return _END;
    }
}
    ''')

def sc(code):
        #strips code
    s = ''
    split = code.split('\n')
    for line in split:
        if not re.match(r'\s*$', line):
            s += line
            s += '\n'
    return s

def nprint(*args, **kwargs):
    print(*args, **kwargs, end='')

def parse_int(string):
    try:
        return int(string)
    except:
        return None

def fix_arb_code(rule_length, arb_code):
    '''
    turns stuff like $1 into a stack offset

    if we have a rule P -> T1 T2 T3
                      p -> T1 T2 T3 NEWP
                                    ^idx
                      P's LEN is 3
    then their offsets are:
                      $$ is just whats at idx
                      $n
                      P -> T(idx-LEN+n-1)
                      so $1 points to T1, $2 to T2 and so on
    '''
    c = arb_code
    template = '(p->symbol_stack.data[( p->symbol_stack.idx - ({rule_len}) + {offset} - 1)])'
    for m in re.finditer(r'\$(\d+)', c):
        off = m.group(1)
        n = parse_int(off)
        if (not n) or (n < 1) or (n > rule_length):
            lrgen.err('$n offset is out of rule length boundary: "{}"'.format(m.group(0)))
        c = c.replace(m.group(0), template.format(offset=off, rule_len=rule_length))
    c = c.replace('$$', '(p->symbol_stack.data[p->symbol_stack.idx])')
    return c

def generate_C_code(table):
    genereate_basic_lexer(table)
    nprint('\n')
    rule_numbers = []

    for i,rule in enumerate(table.index_to_rule):
        #reduce functions return the next state function pointer that  we should goto
        #accept and reject states return null so that the driving loop exits
        nprint(sc('''
static void reduce_{rule_num}_action(struct parse_info_s *p)
{{
        //rule: {rule}
               '''.format(rule_num=i, rule=lrgen.pretty_rule(rule))))

        if len(rule.seq) == 1 and rule.seq[0].val: #copy literal
            nprint('''p->symbol_stack.data[p->symbol_stack.idx].input = '{}';'''.format(rule.seq[0].val[0]))
        code = rule.arbcode.val if rule.arbcode else ''
        nprint('\n',fix_arb_code(len(rule.seq),code))

        if DEBUG:
            nprint('printf("{rule}\\nlhs input: %d\\n",p->symbol_stack.data[p->symbol_stack.idx].input);\n'.format(rule=lrgen.pretty_rule(rule).replace('"','\\"')));
            # nprint('''printf("np:%d\\n user:%d\\n", p->symbol_stack.data[p->symbol_stack.idx].input,p->symbol_stack.data[p->symbol_stack.idx].data.n );''')

        nprint('\n}\n')
        rule_numbers += [rule.rule_num]

    nprint(sc('''
void reduce_action(struct parse_info_s *p, int rule_num){
    switch(rule_num){
    '''))

    for rule_num in rule_numbers:
        nprint(sc('''
        case {rule_num}:
            reduce_{rule_num}_action(p);
            break;
        '''.format(rule_num=rule_num)))

    nprint(sc('''
    default:
        runtime_error();
        break;
    }
}
    '''))

    generate_C_main(table)

def generate_C_main(table):
    nprint('\n#include "simple_lr_main.h"')

def parse_cmd_line_args():
    '''
    parses command line args, returns a dictionary of them
    '''
    stdin_linux = '/dev/fd/0'
    parser = argparse.ArgumentParser(description='vcpu assembler')
    parser.add_argument('--in', default=stdin_linux,
            help='input file')
    parser.add_argument('--out', default='p.out',
            help='out file')
    parser.add_argument('-v', action='store_true')

    args = vars(parser.parse_args())
    for i in range(sys.argv.count('-v')):
        global DEBUG
        DEBUG += 1
    return args

def main():
    global TSRC

    args = parse_cmd_line_args()
    input_str = open(args['in']).read()
    lrgen.TSRC = lrgen.token_source(input_str)

    ast   = lrgen.grammar() #returns a tree like dictionary (SimpleNamespace tree)
    bnf   = lrgen.extract_bnf(ast) #bnf, contains .productions and .syms (grammar and symbol table)
    ctx   = lrgen.create_states(bnf) #(contains .items (states))
    table = lrgen.generate_slr_table(ctx) #turns the .items into a table
    #uses table to generate a parser
    generate_C_data(table)
    generate_C_code(table)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed Apr  4 07:12:45 2018

@author: Daniel Wehner
"""

import numpy as np
import matplotlib.pyplot as plt

from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier

from sklearn.datasets import make_classification
from sklearn.metrics import precision_recall_curve
from sklearn.model_selection import train_test_split

# create data set
X, y = make_classification(n_samples=1250, n_features=3, n_redundant=0,
                           n_informative=3, n_clusters_per_class=4,
                           random_state=50)
plt.scatter(X[:, 0], X[:, 1], marker='o', c=y, s=25, edgecolor='k')
plt.show()

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

# train support vector machine
svc = SVC(gamma=0.05).fit(X_train, y_train)

# calculate precision and recall curve for svm
precision, recall, thresholds = precision_recall_curve(
        y_test, svc.decision_function(X_test))

close_zero = np.argmin(np.abs(thresholds))
plt.plot(precision[close_zero], recall[close_zero], "o", markersize=10,
         fillstyle="none", label="svm threshold 0", mew=2)

plt.plot(precision, recall, label="svm")

# train random forest classifier
rf = RandomForestClassifier(n_estimators=500, random_state=0, max_features=2)
rf.fit(X_train, y_train)

precision_rf, recall_rf, thresholds_rf = precision_recall_curve(
        y_test, rf.predict_proba(X_test)[:, 1])

close_default_rf = np.argmin(np.abs(thresholds_rf - 0.5))
plt.plot(precision_rf[close_default_rf], recall_rf[close_default_rf],
         "^", markersize=10, fillstyle="none", label="rf threshold 0.5", mew=2)

plt.plot(precision_rf, recall_rf, label="rf")

plt.xlabel("Recall")
plt.ylabel("Precision")


"""
Implements the Pyramid Views for PyramAPP requests
"""

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPNotFound

import logging
log = logging.getLogger(__name__)

from pyramapp.models import DBSession, PageRequests

class PyramAppViewRequests(object):
    """Defines the views for requests stored in DB on PyramAPP """
    def __init__(self, request):
        self.request = request

    @view_config(route_name='requests', renderer='templates/requests.jinja2')
    def requests(self):
        """Shows all logs

        Returns:
            dict: A dictionary for template
        """
        query = DBSession.query(PageRequests).order_by(PageRequests.id)

        rdict = { 'requests': []}
        for r in query:
            entry = { 'id': r.id,
                      'sid': r.sid,
                      'path': r.path,
                      'date': r.date }
            rdict['requests'].append(entry)

        return rdict

    @view_config(route_name='requestsid', renderer='templates/request.jinja2')
    def requestsid(self):
        """Shows a single log entry by ID

        Returns:
            dict: A dictionary for template
        """
        req_id = self.request.matchdict['request_id']
        q = DBSession.query(PageRequests).filter(PageRequests.id == req_id).first()

        if q is None:
            return HTTPNotFound('Requested ID {} not found'.format(req_id))

        rdict = { 'r_id': q.id,
                 'r_sid': q.sid,
                 'r_path': q.path,
                 'r_date': q.date
               }

        return rdict