from math import *
from random import *
import pygame
pygame.init()
win=pygame.display.set_mode((1800,900))
S1=pygame.Surface((200,100))
S2=pygame.image.load("MAP.xcf")
run=True
font=pygame.font.SysFont("papyrus",20)
Sags=pygame.Surface((1,100))
Sags.set_colorkey((0,0,0))
Sags.set_alpha(100)
Glassground=pygame.Surface((200,100))
Glassground.set_alpha(1)
Glassground.fill((0,228,255))
Map=[]
light_mode=True
for i in range(50):
    line=[]
    for i1 in range(50):
        if S2.get_at((i,i1))==(255,255,255):
            line.append(0)
        else:
            line.append(1)
    Map.append(line)
x=84
y=897
angle=0
bounced=False
btimer=0
sprint_timer=0
#Inventory Item indexes and descriptions
# 0. Flashlight, (power)
#
#
#
#
inventory=[[0,3000]]
hand_item=0
flashlight_on=False
pygame.mouse.set_visible(False)
t1=False
alive=True
death_reason="Undefined"
hp=100
while run and alive and hp>0:
    for event in pygame.event.get():
        if event.type==pygame.QUIT:
            run=False
    keys=pygame.key.get_pressed()
    if keys[pygame.K_ESCAPE]: run=False
    mouse_rel=pygame.mouse.get_pos()
    mouse_down=pygame.mouse.get_pressed()
    pygame.mouse.set_pos((900,450))
    angle+=(mouse_rel[0]-900)/100
    S1.fill((0,0,0))
    win.blit(pygame.transform.scale(S2,(900,900)),(900,0))
    sec_rays=[]
    for i in range(200):
        angle2=angle+i/100-1
        distance=2
        walkdistance=2
        ray_x=x
        ray_y=y
        rayspeed=[cos(angle2)/2,sin(angle2)/2]
        color=(255,255,255)
        difrakcija=4
        cbonus=[0,0,0]
        dislimit=102
        if 50<i<150 and flashlight_on:
            dislimit=204
        while color in [(255,255,255),(0,228,255),(0,0,1),(0,0,2),(0,0,3),(0,0,4)] and distance<dislimit:
            try:
                color=S2.get_at((round(ray_x),round(ray_y)))
            except:
                break
            if color==(0,228,255):
                sec_rays.append([i,40-150/distance,(0,228,255)])
                rayspeed=[-i for i in rayspeed]
            if color==(0,0,1):
                ray_x+=38
                ray_y+=14
            if color==(0,0,2):
                ray_x-=38
                ray_y-=14
            if color==(0,0,3):
                ray_x-=4
                ray_y-=209
                rayspeed=[-i for i in rayspeed]
            if color==(0,0,4):
                ray_x+=2
                ray_y+=209
                rayspeed=[-i for i in rayspeed]
            ray_x+=rayspeed[0]
            ray_y+=rayspeed[1]
            distance+=1
            walkdistance+=1
        if color==(255,255,255):
            color=(0,0,0)
        if i==100 and walkdistance>4:
            if keys[pygame.K_UP] or keys[pygame.K_w]:
                x+=cos(angle)*0.05
                y+=sin(angle)*0.05
            if keys[pygame.K_SPACE] and sprint_timer<1000:
                x+=cos(angle)*0.15
                y+=sin(angle)*0.15
                sprint_timer+=3
            elif sprint_timer>0:
                sprint_timer-=1
        elif walkdistance<=4:
            x-=cos(angle)*0.05
            y-=sin(angle)*0.05
        distance=40-150/distance
        color=[min(255,max(0,color[i]+cbonus[i])) for i in range(3)]
        try:
            if light_mode:
                if dislimit==204:
                    disbonus=abs(100-i)/24
                    pygame.draw.line(S1,(max(color[0]-distance*(1.5+disbonus),0),max(color[1]-distance*(1.5+disbonus),0),max(color[2]-distance*(1.5+disbonus),0)),(i,distance),(i,100-distance))
                else:
                    pygame.draw.line(S1,(max(color[0]-distance*3.5,0),max(color[1]-distance*3.5,0),max(color[2]-distance*3.5,0)),(i,distance),(i,100-distance))
            else:
                if dislimit==204:
                    disbonus=abs(100-i)/24
                    pygame.draw.line(S1,(max(color[0]-distance*(3.5+disbonus),0),max(color[1]-distance*(3.5+disbonus),0),max(color[2]-distance*(3.5+disbonus),0)),(i,distance),(i,100-distance))
                else:
                    pygame.draw.line(S1,(max(color[0]-distance*5.5,0),max(color[1]-distance*5.5,0),max(color[2]-distance*5.5,0)),(i,distance),(i,100-distance))
        except:
            pass
    for i in range(len(sec_rays)):
        c2=(min(255,max(sec_rays[i][2][0]-sec_rays[i][1]*5,0)),min(255,max(sec_rays[i][2][1]-sec_rays[i][1]*5,0)),min(255,max(sec_rays[i][2][2]-sec_rays[i][1]*5,0)))
        Sags.fill((0,0,0))
        Sags.set_alpha(int(max(0,30-sec_rays[i][1]))*6)
        pygame.draw.line(Sags,c2,(0,sec_rays[i][1]),(0,100-sec_rays[i][1]))
        S1.blit(Sags,(sec_rays[i][0],0))
    pcolor=S2.get_at((round(x),round(y)))
    if pcolor==(0,228,255):
        angle+=pi
        bounced=not bounced
        btimer=0
        t1=True
    elif pcolor==(0,0,1):
        x+=38
        y+=14
        t1=True
    elif pcolor==(0,0,2):
        x-=38
        y-=14
        t1=True
    elif pcolor==(0,0,3):
        x-=4
        angle+=pi
        y-=209
        t1=True
    elif pcolor==(0,0,4):
        x+=2
        angle+=pi
        y+=209
        t1=True
    elif t1:
        t1=False
    else:
        if bounced:
            btimer+=1
            if btimer>800:
                alive=False
                death_reason="Mirror"
            Glassground.set_alpha(int(btimer/4))
            S1.blit(Glassground,(0,0))
        true_x=(x+0.5)*18
        true_y=(y+0.5)*18
        if len(inventory)>0:
            item=inventory[hand_item]
            if item[0]==0: #Flashlight
                if mouse_down[0]:
                    item[1]-=1
                    flashlight_on=True
                else:
                    flashlight_on=False
                if item[1]<=0:
                    inventory.remove(item)
                    flashlight_on=False
                pygame.draw.rect(S1,(255,255,0),(198,25,1,50*item[1]/3000))
        pygame.draw.rect(S1,(255,0,255),(199,25,2,50-50*sprint_timer/1000))
        pygame.draw.rect(S1,(0,255,255),(199,75,2,25-25*btimer/800))

        pygame.draw.circle(win,(255,0,0),(true_x+900,true_y),5)
        win.blit(pygame.transform.scale(S1,(1800,900)),(0,0))
        pygame.display.update()
if alive==False:
    if death_reason=="Mirror":
        S1.blit(font.render("Tu tiki Spogulots",1,(0,114,125)),(30,30))
        win.blit(pygame.transform.scale(S1,(1800,900)),(0,0))
    while run:
        for event in pygame.event.get():
            if event.type==pygame.QUIT:
                run=False
        pygame.display.update()
pygame.quit()


import cv2
from cv2 import HOUGH_GRADIENT
import numpy as np
from matplotlib import pyplot as plt

sct_img = cv2.imread('coins.png')

def nothing(x):
    pass

H = 103
S = 255
V = 255
Hl = 0
Sl = 191
Vl = 119

while True:

    hsv = cv2.cvtColor(sct_img, cv2.COLOR_RGB2HSV)

    cv2.imshow('image',hsv)

    lower_blue = np.array([Hl,Sl,Vl])
    upper_blue = np.array([H,S,V])

    mask = cv2.inRange(hsv, lower_blue, upper_blue)

    # Find the circle blobs on the binary mask:
    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    # Use a list to store the center and radius of the target circles:
    detectedCircles = []

    for i, c in enumerate(contours):

        # Approximate the contour to a circle:
        (x, y), radius = cv2.minEnclosingCircle(c)

        # Compute the center and radius:
        center = (int(x), int(y))
        radius = int(radius)

        if radius > 20 and radius < 40:
            # Draw the circles:
            cv2.circle(sct_img, center, radius, (0, 0, 255), 2)
            cv2.rectangle(sct_img, (center[0] - 5, center[1] - 5), (center[0] + 5, center[1] + 5), (0, 128, 255), -1)

            # Store the center and radius:

            detectedCircles.append([center, radius])


from typing import Any, Dict, Optional, Tuple
import xml.etree.ElementTree as ET

import chc.util.IndexedTable as IT

def has_control_characters(s: str) -> bool:
    for c in s:
        if ord(c) < 32 or ord(c) > 126:
            return True

    else:
        return False

def byte_to_string(b: int) -> str:
    return "{:02x}".format(b)

def value_from_hex(s: str) -> int:
    return int(s, 16)

def hexstring(s: str) -> str:
    result = ""
    for c in s:
        result += byte_to_string(ord(c))
    return result

def dehexstring(h: str) -> str:
    h = h[:]
    result = ""
    try:
        for i in range(int(len(h) / 2)):
            result += chr(int(h[:2], 16))
            h = h[2:]
        return result
    except BaseException:
        print("Error in dehexing string: " + h)
        exit(1)

def decode(ishex: bool, h: str) -> str:
    if ishex:
        return dehexstring(h)
    else:
        return h

def encode(s: str) -> Tuple[bool, str]:
    if has_control_characters(s):
        return (True, hexstring(s))
    else:
        return (False, s)

class IndexedTableError(Exception):
    def __init__(self, msg: str) -> None:
        self.msg = msg

    def __str__(self) -> str:
        return self.msg

class StringIndexedTable(IT.IndexedTableSuperclass):
    def __init__(self, name: str) -> None:
        IT.IndexedTableSuperclass.__init__(self, name)
        self.stringtable: Dict[str, int] = {}  # string -> index
        self.indextable: Dict[int, str] = {}  # index -> string
        self.next = 1

    def reset(self) -> None:
        self.stringtable = {}
        self.indextable = {}
        self.next = 1

    def add(self, s: Optional[str]) -> int:
        if s is None:
            print("Attempt to index None in string table")
            raise IndexedTableError(self.name + ": Attempt to index None")
        if s in self.stringtable:
            return self.stringtable[s]
        else:
            index = self.next
            self.stringtable[s] = index
            self.indextable[index] = s
            self.next += 1
            return index

    def size(self) -> int:
        return self.next - 1

    def retrieve(self, index: int) -> str:
        if index in self.indextable:
            return self.indextable[index]
        else:
            msg = (
                "Unable to retrieve item "
                + str(index)
                + " from table "
                + self.name
                + " (size: "
                + str(self.size())
                + ")"
            )
            raise IndexedTableError(
                msg + "\n" + self.name + ", size: " + str(self.size())
            )

    def read_xml(self, node: Optional[ET.Element]) -> None:
        if node is None:
            print("Xml node not present in string table")
            raise IndexedTableError("string table")
        for snode in node.findall("n"):
            xml_ix = snode.get("ix")
            if xml_ix is None:
                raise IndexedTableError("`ix` missing from element")
            index = int(xml_ix)
            ishex = snode.get("hex", "no") == "yes"
            xml_v = snode.get("v")
            if xml_v is None:
                raise IndexedTableError("`v` missing from element")
            s = decode(ishex, xml_v)
            self.stringtable[s] = index
            self.indextable[index] = s
            if index >= self.next:
                self.next = index + 1

    def write_xml(self, node: ET.Element) -> None:
        for index in sorted(self.indextable):
            s = self.indextable[index]
            (ishex, sencoded) = encode(s)
            snode = ET.Element("n")
            snode.set("v", sencoded)
            snode.set("ix", str(index))
            node.append(snode)

    def __str__(self) -> str:
        lines = []
        lines.append("\nstring-table")
        for ix in sorted(self.indextable):
            lines.append(str(ix).rjust(4) + " " + str(self.indextable[ix]))
        return "\n".join(lines)

if __name__ == "__main__":

    print(str(has_control_characters("\n")))
    print(str(has_control_characters("string")))

    print(hexstring("\n\n"))
    print(dehexstring("0a0a"))

    print(decode(*encode("string")))
    print(decode(*encode("\n\n")))

    print(dehexstring("4d4158504154484c454e3d25640a"))
    print(
        dehexstring(
            "496e7075742070617468203d2025732c207374726c656e287061746829203d2025640a"
        )
    )
    print(dehexstring("4d4158504154484c454e203d2025640a"))


"""
Author:     Brian Mascitello
Date:       12/16/2017
Websites:   http://adventofcode.com/2017/day/14
Info:       --- Day 14: Disk Defragmentation ---
            --- Part Two ---
"""

import copy
from functools import reduce

def construct_dense_hash(sparse_hash):
    constructed_hash = list()
    groups_of_sixteen = [sparse_hash[index:index + 16] for index in range(0, len(sparse_hash), 16)]
    for group in groups_of_sixteen:
        xor = reduce((lambda x, y: x ^ y), group)
        constructed_hash.append(xor)
    return constructed_hash

def determine_ordinals_list(input_data):
    data_length_list = list()
    for item in input_data:
        temp_list = list()
        for char in item:
            temp_list.append(ord(char))
        temp_list.extend([17, 31, 73, 47, 23])
        data_length_list.append(temp_list)
    return data_length_list

def find_valid_neighbors(x, y, maximum):
    neighbors = set()
    neighbors.add((x - 1, y))
    neighbors.add((x + 1, y))
    neighbors.add((x, y - 1))
    neighbors.add((x, y + 1))
    remove_set = neighbors.copy()
    for coordinates in neighbors:
        for position in coordinates:
            if position < 0 or position >= maximum:
                if coordinates in remove_set:
                    remove_set.remove(coordinates)
    return remove_set

def get_data(input_text):
    with open(input_text) as file:
        data_from_file = file.read()
    file.close()
    return data_from_file

def knot_hash(input_list, length_list, rounds):
    copy_input_list = copy.copy(input_list)
    current_position = 0
    skip_size = 0
    while rounds:
        for item in length_list:
            end_position = (current_position + item) % len(copy_input_list)
            if current_position <= end_position:
                copy_input_list[current_position: end_position] = list(
                    reversed(copy_input_list[current_position: end_position]))
            else:
                sub_list = copy_input_list[current_position:]
                sub_list_first_size = len(sub_list)
                sub_list.extend(copy_input_list[:end_position])
                sub_list.reverse()  # Reverses the sub_list before modifying input_list values.
                copy_input_list[current_position:] = sub_list[:sub_list_first_size]
                copy_input_list[:end_position] = sub_list[sub_list_first_size:]
            current_position = (end_position + skip_size) % len(copy_input_list)
            skip_size += 1
        rounds -= 1
    return copy_input_list

def list_to_hex_string(list_of_numbers):
    hex_str = ''
    for number in list_of_numbers:
        hex_str += hex(number)[2:].zfill(2)  # Forward fill hex with 0 if too small.
    return hex_str

def make_data_list(input_data, rounds):
    output_list = list()
    for index in range(0, rounds):
        output_list.append(input_data + '-' + str(index))
    return output_list

standard_list = list(range(256))
# data = 'flqrgnkx'  # test 8108
data = 'jxqlasbh'
data_list = make_data_list(data, 128)
lengths_list = determine_ordinals_list(data_list)

knots_list = list()
for length in lengths_list:
    knots_list.append(knot_hash(standard_list, length, 64))

dense_list = list()
for knot in knots_list:
    dense_list.append(construct_dense_hash(knot))

hex_list = list()
for dense in dense_list:
    hex_list.append(list_to_hex_string(dense))

binary_list = list()
for hex_value in hex_list:
    temp_bin = list()
    for character in hex_value:
        # https://stackoverflow.com/questions/1425493/convert-hex-to-binary
        temp_bin.append(bin(int(character, 16))[2:].zfill(4))
    binary_list.append(''.join(temp_bin))

ones_locations = set()
for row, line in enumerate(binary_list):
    for column, character in enumerate(line):
        if character == '1':
            ones_locations.add((row, column))

regions = 0
while ones_locations:
    new_region = [ones_locations.pop()]
    regions += 1
    while new_region:
        old_locations = new_region.copy()
        new_region = list()
        for row, column in old_locations:
            for pair in find_valid_neighbors(row, column, len(binary_list)):
                if pair in ones_locations:
                    ones_locations.remove(pair)
                    new_region.append(pair)

print(regions)  # 1182

Advent of Code 2017/Day 14 2017/Day14Q2 2017.py


import os
from collections import Counter
from string import punctuation
import glob
import re

class Transcript:
    """This class is a representation of a transcript following the norm from
    the CHILDES database. As long as the raw transcript file follows that norm,
    all kinds of informations and extractions from the transcript should be
    obtainable through the class methods and variables."""

    def __init__(self, filepath):
        try:
            file = open(filepath, 'r', encoding='utf-8')
            self.name = os.path.basename(file.name)

            # store the raw transcript, but clean it a little bit
            self.raw_transcript = file.read()
            remove_list = ['\t', '\r']
            for item in remove_list:
                self.raw_transcript = self.raw_transcript.replace(item, '')

            # extract headerlines and transcriptlines
            text = self.raw_transcript.split('\n')
            self.headers = []
            self.lines = []
            for line in text:
                if line.startswith('@'):
                    self.headers.append(line)
                elif line.startswith('*') or line.startswith('%'):
                    self.lines.append(line)
                else:  # continuation of previous line
                    if not self.lines:
                        self.headers[-1] = self.headers[-1] + line
                    else:
                        self.lines[-1] = self.lines[-1] + line
            self.fully_loaded = True  # flag that transcript is fully loaded

        except IOError as e:
            self.fully_loaded = False  # flag that transcript was not loaded
            print('An error occured when loading:', filepath)
            print('Error message:', e)

    def lines_as_tuples(self, speakers='all', annotations=False,
                        as_blocks=False):
        """Return a list of tuples of all utterance lines, where tuple[0] is
        the three letter initials for the speaker and tuple[1] is the line. One
        or more speakers can be specified to retrieve just lines by these and
        one or more flags can be marked to get annotations for the requested
        speaker(s). If as_blocks is flagged, the lines along with their
        annotations are passed in lists."""

        if speakers == 'all':
            speakers = self.speakers()
        if type(speakers) == str:
            speakers = [speakers]

        # check if the requested speakers are present in the transcript
        # and report if they are not
        for speaker in speakers:
            if speaker not in self.speakers():
                print(f'WARNING: The speaker {speaker} is not present ' +
                      f'in the transcript {self.name}.')

        # make list with lines as three part tuples
        tuples = [(line[0], line[1:4], line[5:]) for line in self.lines]

        # divide into blocks of turns with their annotations
        blocks = []
        for line in tuples:
            if line[0] == '*':
                blocks.append([(line[1], line[2])])
            elif line[0] == '%' and annotations == True:
                blocks[-1].append((line[1], line[2]))

        blocks = [block for block in blocks if block[0][0] in speakers]

        if as_blocks:
            return blocks

        # put together the blocks of the requested speakers with annotations
        # if requested
        tuples = []
        for block in blocks:
            if block[0][0] in speakers and not annotations:
                tuples += [line for line in block if line[0] in speakers]
            elif block[0][0] in speakers and  annotations:
                tuples += [line for line in block]

        return tuples

    def tokens(self, speakers='all'):
        """Return a list of tokens uttered by the specified speaker(s). If no
        speakers are specified, return tokens for all speakers."""

        if speakers == 'all':
            speakers = self.speakers()
        if type(speakers) == str:
            speakers = [speakers]

        # get tokens from the specified speakers
        tuples = self.lines_as_tuples(speakers)
        tokens = [word.lower() for tpl in tuples for word in tpl[1].split()]

        # clean for punctuation
        tokens = ' '.join(tokens)
        tokens = ''.join(c for c in tokens if c not in punctuation)
        tokens = tokens.split()

        return tokens

    def types(self, speakers='all'):
        """Return a list of types uttered by the specified speaker(s). If no
        speakers are specified, return types for all speakers."""

        return set(self.tokens(speakers=speakers))

    def ttr(self, speakers='all', disregard=()):
        """Return the type-to-token-ratio of the transcript in whole. Pass
        specific speaker(s) to get it for only that/these speaker(s). A list of
        words to be disregarded in the calculation, e.g. function words, can be
        passed if needed."""

        tokens = [word for word in self.tokens(speakers=speakers)
                  if word not in disregard]
        types = set(tokens)

        return len(types) / len(tokens)

    def mlu(self, speaker='CHI', disregard=('www', 'yyy', 'xxx')):
        """Return the MLU for the given speaker, the target child as default,
         in the transcript. """

        blocks = self.lines_as_tuples(speakers=speaker, annotations=True,
                                      as_blocks=True)

        # filter out utterances containing the disregarded words
        lines = []
        for block in blocks:
            unclear = False
            for word in disregard:
                if word in block[0][1]:
                    unclear = True
            if not unclear:
                lines += block

        annotation = [clean_line(line[1]) for line in lines if line[0] == 'mor']
        morphemes = []
        for line in annotation:
            words = line.split()
            for word in words:
                word = re.split('[-~#]', word)
                morphemes += word

        return len(morphemes) / len(annotation)

    def word_freqs(self, speakers='all'):
        """Return a Counter object of tokens uttered by the specified
        speaker(s). If no speakers are specified, return a Counter object for
        all speakers."""

        return Counter(self.tokens(speakers=speakers))

    def prop_word_freqs(self, speakers='all'):
        """Return a dict of words and their proportional frequencies."""

        # get number of tokens and a list of tuples with words and frequencies
        freqs = self.word_freqs(speakers=speakers)
        tokens = sum(freqs.values())
        freqs = freqs.most_common()

        # make a dict with the word as key and prop freq as value
        prop_freqs = {word[0]: word[1]/tokens for word in freqs}

        return prop_freqs

    def speakers(self):
        """Return a list of all speakers that appear in the transcript"""

        return list({line[1:4] for line in self.lines if line.startswith('*')})

    def speaker_details(self):
        """Return a dictionary of dictionaries containing details about the
        given speaker(s). If no info is given in the original transcript file
        on some details, e.g. age or sex, those entries will simply be empty.
        The entries are: lang, corp, name, age, sex, role. As an example, the
        child's age is called by transcript.speaker_details()['CHI']['age']"""

        # find the ID lines from the header lines and split these
        ids = [id_str for id_str in self.headers if id_str.startswith('@ID')]
        ids = [entry[4:].split(sep='|') for entry in ids]

        # assign the values to their respective dict entries
        ids = [{'lang': entry[0], 'corp': entry[1], 'name': entry[2],
                'age': entry[3], 'sex': entry[4], 'role': entry[7]}
               for entry in ids]

        # create a dict with names as keys and the dicts as values
        ids = {entry['name']: entry
               for entry in ids if entry['name'] in self.speakers()}

        return ids

    def children(self):
        """Return a list of the target child(ren) in the transcript."""

        children = [entry['name'] for entry in self.speaker_details().values()
                    if entry['role'] == 'Target_Child']

        return children

def load_all_from_dir(dirname):
    """Return a list of Transcript objects loaded from the given directory
    sorted after file names. The directory name should be stated either as
    relative path from the working directory or as an absolute path."""

    prev_dir = os.getcwd()
    os.chdir(dirname)

    # load all transcripts from the folder and clean out non-loaded ones
    trans = [Transcript(file) for file in glob.glob('*.cha')]
    trans = [trn for trn in trans if trn.fully_loaded]

    # make sure the list is sorted
    trans.sort(key=lambda x: x.name)

    os.chdir(prev_dir)

    return trans

def age_in_months(age):
    """Return an age passed in the format y;mm.dd as the number of months with
    two decimal numbers."""

    # split the passed age string at the specified characters
    y_md = age.split(';')
    m_d = y_md[1].split('.')

    # convert each number to a float
    years = float(y_md[0])
    months = float(m_d[0])
    # in case days is not specified, assign 0
    try:
        days = float(m_d[1])
    except:
        days = 0

    # calculate number of months
    total = years * 12 + months + days / 30

    return float(f'{total:.2f}')

def clean_line(line):
    """Clean a string for ' .', ' ?', ' !'"""

    remove_list = [' .', ' ?', ' !']
    for item in remove_list:
        line = line.replace(item, '')

    return line

def word_freqs_all(transcripts, speakers='all'):
    """Return one Counter object of all transcripts passed counting only
    utterances from the specified speaker(s)."""

    counters = [trn.word_freqs(speakers=speakers) for trn in transcripts]
    counter_all = Counter()
    for counter in counters:
        counter_all.update(counter)

    return counter_all

def basic_stats(transcript: Transcript, speakers='CHI'):
    """Return a tuple containing age in months, number of tokens, number of
    types and TTR."""

    age = age_in_months(transcript.speaker_details()['CHI']['age'])
    tokens = len(transcript.tokens(speakers='CHI'))
    types = len(transcript.types(speakers='CHI'))
    ttr = transcript.ttr(speakers=speakers)

    return age, tokens, types, ttr


"""
Scraper for fire alerts in Los Angeles

http://groups.google.com/group/LAFD_ALERT/
RSS: http://groups.google.com/group/LAFD_ALERT/feed/rss_v2_0_msgs.xml?num=50
"""

from ebdata.retrieval.scrapers.base import ScraperBroken
from ebdata.retrieval.scrapers.list_detail import RssListDetailScraper, SkipRecord
from ebdata.retrieval.scrapers.newsitem_list_detail import NewsItemListDetailScraper
from ebpub.db.models import NewsItem
import datetime
import re

class AlertScraper(NewsItemListDetailScraper, RssListDetailScraper):
    schema_slugs = ('fire-alerts',)
    has_detail = False
    sleep = 4

    def list_pages(self):
        yield self.get_html('http://groups.google.com/group/LAFD_ALERT/feed/rss_v2_0_msgs.xml?num=50')

    def clean_list_record(self, rss_record):
        record = {
            'pub_date': datetime.date(*rss_record.pop('updated_parsed')[:3]),
            'summary': rss_record['summary'].strip(),
        }
        if re.search(r'^(?i)\*UPDATE:', record['summary']):
            m = re.search(r'^\*UPDATE:\s*(?P<location_name>[^\*]*)\*\s*(?P<description>.*)\s*-\s*(?P<reporter>.*?)\#\#\#$', record['summary'])
            if not m:
                self.logger.warn('Could not parse update %r' % record['summary'])
                raise SkipRecord('Could not parse update %r' % record['summary'])
            record.update(m.groupdict())
            record.update({
                'is_update': True,
                'incident_type': '',
                'fire_station': '',
                'radio_channels': '',
                'incident_time': '',
            })
        else: # Not an update
            m = re.search(r'^\*(?P<incident_type>[^\*]*)\*\s*(?P<location_name>[^;]*);\s*MAP (?:\d+[- ]\w\d)?;\s*FS (?P<fire_station>\d+); (?P<description>.*?); Ch:(?P<radio_channels>[\d, ]+)\s*@(?P<incident_time>\d\d?:\d\d [AP]M)?\s*-(?P<reporter>.*?)\#\#\#$', record['summary'])
            if not m:
                raise SkipRecord('Could not parse %r' % record['summary'])
            record.update(m.groupdict())
            record['incident_type'] = record['incident_type'].upper() # Normalize
            record['radio_channels'] = ','.join(record['radio_channels'].split(','))
            record['is_update'] = False
        record['description'] = record['description'].replace(' ', ' ').replace('"', '"').replace('&', '&').strip()
        record['location_name'] = record['location_name'].strip()

        # Get the incident ID and message ID from the Google Groups URL.
        # We'll use these as unique identifiers.
        m = re.search(r'browse_thread/thread/(?P<incident_id>[^/]*)/(?P<message_id>[^\?]*)\?', rss_record['link'])
        if not m:
            raise ScraperBroken('Got weird URL: %r', rss_record['link'])
        record.update(m.groupdict())
        record['link'] = rss_record['link']

        # I can't figure out why this record is causing errors, so for now
        # we'll just skip it.
        if record['message_id'] == '0faabeab3aad8492':
            raise SkipRecord()

        return record

    def existing_record(self, record):
        try:
            qs = NewsItem.objects.filter(schema__id=self.schema.id)
            qs = qs.by_attribute(self.schema_fields['message_id'], record['message_id'])
            return qs[0]
        except IndexError:
            return None

    def save(self, old_record, list_record, detail_record):
        if old_record is not None:
            return
        incident_type = self.get_or_create_lookup('incident_type', list_record['incident_type'], list_record['incident_type'], make_text_slug=False)
        reporter = self.get_or_create_lookup('reporter', list_record['reporter'], list_record['reporter'])
        fire_station = self.get_or_create_lookup('fire_station', list_record['fire_station'], list_record['fire_station'])
        attributes = {
            'incident_type': incident_type.id,
            'description': list_record['summary'],
            'reporter': reporter.id,
            'fire_station': fire_station.id,
            'incident_time': list_record['incident_time'],
            'incident_id': list_record['incident_id'],
            'message_id': list_record['message_id'],
        }
        if list_record['is_update']:
            title = 'Update' # TODO: Better title that takes into account the incident type.
        else:
            title = incident_type.name
        self.create_newsitem(
            attributes,
            title=title,
            description=list_record['description'],
            url=list_record['link'],
            item_date=list_record['pub_date'],
            location_name=list_record['location_name'],
        )

if __name__ == "__main__":
    from ebdata.retrieval import log_debug
    AlertScraper().update()


# encoding: utf-8

import argparse
import os
import sys
import torch

from torch.backends import cudnn

sys.path.append('.')
from config import cfg
from data import make_data_loader
from engine.trainer import do_train, do_train_with_center
from modeling import build_model
from layers import make_loss, make_loss_with_center
from solver import make_optimizer, make_optimizer_with_center, WarmupMultiStepLR

from utils.logger import setup_logger

def train(cfg):
    # prepare dataset
    train_loader, val_loader, num_query, num_classes, clustering_loader = make_data_loader(cfg)

    # prepare model
    model = build_model(cfg, num_classes)

    if cfg.MODEL.IF_WITH_CENTER == 'on':
        loss_func, center_criterion_part, center_criterion_global, center_criterion_fore = make_loss_with_center(cfg, num_classes)
        optimizer, optimizer_center = make_optimizer_with_center(cfg, model, center_criterion_part, center_criterion_global, center_criterion_fore)
    else:
        loss_func = make_loss(cfg, num_classes)
        optimizer = make_optimizer(cfg, model)

    # Add for using self trained model
    if cfg.MODEL.PRETRAIN_CHOICE == 'imagenet':
        start_epoch = 0
        scheduler = WarmupMultiStepLR(optimizer, cfg.SOLVER.STEPS, cfg.SOLVER.GAMMA, cfg.SOLVER.WARMUP_FACTOR,
                                      cfg.SOLVER.WARMUP_ITERS, cfg.SOLVER.WARMUP_METHOD)
    else:
        print('Only support pretrain_choice for imagenet, but got {}'.format(cfg.MODEL.PRETRAIN_CHOICE))

    if cfg.MODEL.IF_WITH_CENTER == 'on':
        do_train_with_center(
            cfg,
            model,
            center_criterion_part,
            center_criterion_global,
            center_criterion_fore,
            train_loader,
            val_loader,
            optimizer,
            optimizer_center,
            scheduler,      # modify for using self trained model
            loss_func,
            num_query,
            start_epoch,     # add for using self trained model
            clustering_loader
        )
    else:
        do_train(
            cfg,
            model,
            train_loader,
            val_loader,
            optimizer,
            scheduler,      # modify for using self trained model
            loss_func,
            num_query,
            start_epoch,     # add for using self trained model
            clustering_loader
        )

def main():
    parser = argparse.ArgumentParser(description="ReID Baseline Training")
    parser.add_argument(
        "--config_file", default="", help="path to config file", type=str
    )
    parser.add_argument("opts", help="Modify config options using the command-line", default=None,
                        nargs=argparse.REMAINDER)

    args = parser.parse_args()

    num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1

    if args.config_file != "":
        cfg.merge_from_file(args.config_file)
    cfg.merge_from_list(args.opts)
    cfg.freeze()

    output_dir = cfg.OUTPUT_DIR
    if output_dir and not os.path.exists(output_dir):
        os.makedirs(output_dir)

    logger = setup_logger("reid_baseline", output_dir, 0)
    logger.info("Using {} GPUS".format(num_gpus))
    logger.info(args)

    if args.config_file != "":
        logger.info("Loaded configuration file {}".format(args.config_file))
        with open(args.config_file, 'r') as cf:
            config_str = "\n" + cf.read()
            logger.info(config_str)
    logger.info("Running with config:\n{}".format(cfg))

    if cfg.MODEL.DEVICE == "cuda":
        os.environ['CUDA_VISIBLE_DEVICES'] = cfg.MODEL.DEVICE_ID    # new add by gu
    cudnn.benchmark = True
    train(cfg)

if __name__ == '__main__':
    main()


from ExtendDT import ext_dt
from Sequence import *

#> Find the optimal sequence for a single node when its children are optimized
def PruneOptimalSingle(xdt, main_seq, root_id):
    current_seq = copy.copy(main_seq[root_id])
    #* if optimized is not available, make it
    if  not current_seq.optimized:
        #* Begin of optimization procedure ==============================
        print(f"[{root_id} (d:{xdt.node_depth[root_id]})] | Opting ===>")
        # * check this is a leaf
        if xdt.is_leaf[root_id]:
            combined_seq = PruneSequence()
            combined_seq.OptFinished()
            current_seq = combined_seq
        else:
            #* Find children IDs
            left_root = xdt.children[root_id][0]
            right_root = xdt.children[root_id][1]
            #* Get optimal sequence of children
            lef_seq, node_id  = PruneOptimalSingle(xdt, main_seq, left_root)
            right_seq, node_id  = PruneOptimalSingle(xdt, main_seq, right_root)
            #* Combine children sequences with each other
            combined_seq = CombineSequence(lef_seq, right_seq, root_id)
            # * Append combined seq to the sequence of root
            current_seq.AppendSequence(combined_seq)
            #+ Tuple of changing root into a leaf
            current_tuple = [xdt.abs_cost[root_id], xdt.miss_class[root_id], [root_id]]
            #* Append this to the sequence
            current_seq.AppendTuple(current_tuple)
            #* Set register to finished
            current_seq.OptFinished()
        #* End of optimization procedure ==============================
        print(f"[{root_id}] ===> Ready |")
    return current_seq, root_id

#> Function for finding the optimal sequence of a node recursively
def PruneOptimal(xdt, goal_node):
    #* Get depth list
    xdt.getNodeDepth()
    depth_list = xdt.nodes_in_depth
    #* get the cost factors
    xdt.getAbsCostFactor()
    #* Find the children relations
    xdt.getChildren()
    #* Make the empty sequence for all
    main_seq = [PruneSequence()] * xdt.n_all_nodes
    #* Go through all depth from bottom to top
    for depth_row in range(len(depth_list)-1, -1, -1):
        row_list = depth_list[depth_row]
        #* Go through nodes in this depth
        for node_in_list in row_list:
            current_seq, node_id = PruneOptimalSingle(xdt, main_seq, node_in_list)
            main_seq[node_in_list] = current_seq
            if node_in_list == goal_node:
                break
        if node_in_list == goal_node:
            break
    return main_seq[goal_node]

#> Function for finding the optimal sequence of a node recursively
def PruneOptimalParallel(xdt, goal_node):
    #* Load multi-processor pckg
    import multiprocessing as mp
    #* Get depth list
    xdt.getNodeDepth()
    depth_list = xdt.nodes_in_depth
    #* get the cost factors
    xdt.getAbsCostFactor()
    #* Find the children relations
    xdt.getChildren()
    #* Make the empty sequence for all
    main_seq = [PruneSequence()] * xdt.n_all_nodes
    #+ Is goal reached
    goal_reached = False
    #* Start the pool
    num_processor = mp.cpu_count()
    pool = mp.Pool(processes = int(num_processor/2))
    #* Go through all depth from bottom to top
    for depth_row in range(len(depth_list)-1, -1, -1):
        #* Go through nodes in this depth
        row_list = depth_list[depth_row]
        #* Run the pool in this depth
        result_async = [pool.apply_async(PruneOptimalSingle, args = (xdt, main_seq, node_in_list)) for node_in_list in row_list]
        #* Get the pool results
        for worker_idx in result_async:
            current_seq = worker_idx.get()[0]
            current_node = worker_idx.get()[1]
            main_seq[current_node] = current_seq
            if current_node == goal_node:
                goal_reached = True
        if goal_reached:
            break
    pool.close()
    # pool.join()
    return main_seq[goal_node]

#> A function to get the
def get_ORAP_sequence(dt_base, **kwargs):
    # parse optional input
    use_parallel = False # by default use a single core
    for key, value in kwargs.items():
        # if use_parallel passed in set its value
        if key == "use_parallel":
            use_parallel = value
    #* Build the extended tree object
    xdt = ext_dt(dt_base)
    #* Calculate the optimal sequence for the root of the whole DT
    goal_node = 0
    #* Decide about number of processors
    if use_parallel:
        goal_seq = PruneOptimalParallel(xdt, goal_node)
    else:
        goal_seq = PruneOptimal(xdt, goal_node)
    #* Get the sequence for root (complete sequence)
    ORAP_base_sequence = copy.copy(goal_seq)
    #* Sort sequence according to the cost
    sort_factor = "cost"
    ORAP_base_sequence.SortSequence(sort_factor)
    #* Add the original tree to the sequence
    ORAP_base_sequence.cost.insert(0, 0)
    ORAP_base_sequence.miss.insert(0, 0)
    ORAP_base_sequence.removed.insert(0, [])

    return ORAP_base_sequence

#>A function to remove a series of nodes from a base DT
def series_prune(base_dt, node_series):
    pruned_xdt = ext_dt(copy.deepcopy(base_dt))
    #* Step by step remove branches
    for prune_node in node_series:
        current_dt = pruned_xdt.removeBranch(prune_node)
        pruned_xdt = ext_dt(current_dt)
    return pruned_xdt.base_dt


import numpy as np
import os
import cv2
from sklearn.feature_extraction import image
import random
import csv
import shutil

def colors_new_train_patches_to_npy_file():
    path = '/Users/eloymarinciudad/Downloads/colors_new/train'
    label = {'black': 0, 'blue': 1, 'brown': 2, 'green': 3, 'grey': 4, 'orange': 5, 'pink': 6,
             'purple': 7, 'red': 8, 'white': 9, 'yellow': 10}

    labels = []
    first = True
    classes = os.listdir(path)
    for color in classes:
        subdir_class = path + '/' + color
        print(subdir_class)
        if os.path.isdir(subdir_class):
            for imagen in os.listdir(subdir_class):
                if os.path.isfile(subdir_class + '/' + imagen) and imagen.endswith('.jpg'):
                    bgr = cv2.imread(subdir_class + '/' + imagen)

                    img = bgr[:, :, [2, 1, 0]]
                    n = img.shape[0]
                    m = img.shape[1]
                    # mid_ima = img[int(n / 2 - n / 4): int(n / 2 + n / 4), int(m / 2 - m / 4): int(m / 2 + m / 4)]
                    patches = image.extract_patches_2d(img, (32, 32))
                    random_index = random.randrange(len(patches))
                    patch = patches[random_index]

                    r_max = max(patch[:, :, 0].flatten())
                    r_min = min(patch[:, :, 0].flatten())
                    g_max = max(patch[:, :, 1].flatten())
                    g_min = min(patch[:, :, 1].flatten())
                    b_max = max(patch[:, :, 2].flatten())
                    b_min = min(patch[:, :, 2].flatten())

                    count = 0
                    repeat = False
                    if (r_max - r_min >= 100 and g_max - g_min >= 100) or (
                            r_max - r_min >= 100 and b_max - b_min >= 100) or (
                            g_max - g_min >= 100 and b_max - b_min >= 100):
                        repeat = True

                    while repeat:
                        random_index = random.randrange(len(patches))
                        patch = patches[random_index]
                        r_max = max(patch[:, :, 0].flatten())
                        r_min = min(patch[:, :, 0].flatten())
                        g_max = max(patch[:, :, 1].flatten())
                        g_min = min(patch[:, :, 1].flatten())
                        b_max = max(patch[:, :, 2].flatten())
                        b_min = min(patch[:, :, 2].flatten())
                        count += 1
                        if (r_max - r_min >= 100 and g_max - g_min >= 100) or (
                                r_max - r_min >= 100 and b_max - b_min >= 100) or (
                                g_max - g_min >= 100 and b_max - b_min >= 100):
                            repeat = True
                        else:
                            repeat = False
                        if count == 10000:
                            break

                    if count > 50000:
                        print(
                            f'Count: {count}\nR: {r_min} to {r_max}\n G: {g_min} to {g_max}\n B: {b_min} to {b_max}\n')
                        cv2.imwrite(f'test_patch/patch_{imagen}', patch[:, :, [2, 1, 0]])
                    patch = cv2.cvtColor(patch, cv2.COLOR_RGB2LAB)
                    patch = patch / 255
                    img = np.reshape(patch, 32 * 32 * 3)
                    if first:
                        dataset = img
                        first = False
                    else:
                        dataset = np.vstack((dataset, img))
    labels = np.asarray(labels)
    np.save('colors_new_train_patches_data.npy', dataset)
    np.save('colors_new_train_patches_labels.npy', labels)

def split_colors_new_dataset():
    path = "/Users/eloymarinciudad/Downloads/colors_new_original"
    classes = os.listdir(path)

    print(classes)
    num_of_images = 0
    for color in classes:
        subdir_class = path + '/' + color
        print(subdir_class)
        if os.path.isdir(subdir_class):
            for imagen in os.listdir(subdir_class):
                shutil.copy(subdir_class + '/' + imagen,
                            f'/Users/eloymarinciudad/Downloads/colors_new/{color}_{imagen}')
                num_of_images += 1

    dest_path = '/Users/eloymarinciudad/Downloads/colors_new'
    imagenes = os.listdir(dest_path)
    random_index_list = random.sample(range(num_of_images), round(num_of_images * 0.2))
    test_list = random_index_list[:round(len(random_index_list) / 2)]
    val_list = random_index_list[round(len(random_index_list) / 2):]
    ima_index = 0
    for imagen in imagenes:
        if imagen.endswith('.jpg'):
            color = imagen.split(sep='_')[0]

            if ima_index in test_list:
                shutil.move(dest_path + '/' + imagen, dest_path + f'/test/{color}/' + imagen)
            elif ima_index in val_list:
                shutil.move(dest_path + '/' + imagen, dest_path + f'/validation/{color}/' + imagen)
            else:
                shutil.move(dest_path + '/' + imagen, dest_path + f'/train/{color}/' + imagen)

            ima_index += 1

def load_termisk_reduced():
    path = "/content/termisk_dataset"
    # path = "/Users/eloymarinciudad/Downloads/termisk_dataset"
    split_paths = os.listdir(path)
    print(split_paths)
    labels = []
    index_label = 0
    first = True
    for split in split_paths:
        subdir = path + '/' + split
        print(subdir)
        if os.path.isdir(subdir):
            classes = os.listdir(subdir)
            classes = [s for s in classes if '.' not in s]
            print(classes)
            for label in classes:
                class_path = subdir + '/' + label
                count = 0
                if label not in ('17', '3', '0', '1', '2', '4', '7', '9', '12', '13', '14', '16'):
                    for image in os.listdir(class_path):
                        if os.path.isfile(class_path + '/' + image) and image.endswith('.png'):
                            img = cv2.imread(class_path + '/' + image, cv2.IMREAD_GRAYSCALE)
                            img = cv2.resize(img, (64, 64), interpolation=cv2.INTER_AREA)
                            img = np.reshape(img, 64 * 64)
                            img = img / 255
                            labels.append(index_label)
                            if first:
                                dataset = img
                                first = False
                            else:
                                dataset = np.vstack((dataset, img))
                            count += 1
                        if count >= 1650:
                            break
                    index_label += 1
    labels = np.asarray(labels)
    return dataset, labels

def load_colors_new():
    path = "/content/colors_new"
    classes = os.listdir(path)
    logs_file = open('/content/ClusterGAN/color_patches_logs.csv', 'w')
    writer = csv.writer(logs_file)
    writer.writerow(['ima_path', 'patch_index'])
    print(classes)
    labels = []
    index_label = 0
    first = True
    for color in classes:
        subdir_class = path + '/' + color
        print(subdir_class)
        if os.path.isdir(subdir_class):
            for imagen in os.listdir(subdir_class):
                if os.path.isfile(subdir_class + '/' + imagen):
                    bgr = cv2.imread(subdir_class + '/' + imagen)
                    # bgr = cv2.resize(bgr, (128, 128), interpolation=cv2.INTER_AREA)
                    # img = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB)
                    img = bgr[:, :, [2, 1, 0]]
                    n = img.shape[0]
                    m = img.shape[1]
                    mid_ima = img[int(n / 2 - n / 4): int(n / 2 + n / 4), int(m / 2 - m / 4): int(m / 2 + m / 4)]
                    patches = image.extract_patches_2d(mid_ima, (32, 32))
                    random_index = random.randrange(len(patches))
                    patch = patches[random_index]
                    patch = cv2.cvtColor(patch, cv2.COLOR_RGB2LAB)
                    patch = patch / 255
                    img = np.reshape(patch, 32 * 32 * 3)

                    labels.append(index_label)
                    # path, patch_index
                    writer.writerow([subdir_class + '/' + imagen, random_index])
                    if first:
                        dataset = img
                        first = False
                    else:
                        dataset = np.vstack((dataset, img))
            index_label += 1
    labels = np.asarray(labels)
    logs_file.close()
    return dataset, labels

def load_google_colors():
    path = "/content/ClusterGAN/colors/google_colors"
    dirs = os.listdir(path)
    print(dirs)
    labels = []
    index_label = 0
    first = True
    for item in dirs:
        subdir = path + '/' + item
        print(subdir)
        if os.path.isdir(subdir):
            for image in os.listdir(subdir):
                if os.path.isfile(subdir + '/' + image):
                    bgr = cv2.imread(subdir + '/' + image)
                    # img = cv2.cvtColor(img, cv2.COLOR_BGR2LAB)
                    img = bgr[:, :, [2, 1, 0]]
                    img = np.reshape(img, 32 * 32 * 3)
                    img = img / 255
                    labels.append(index_label)
                    if first:
                        dataset = img
                        first = False
                    else:
                        dataset = np.vstack((dataset, img))
            index_label += 1
    labels = np.asarray(labels)
    return dataset, labels

def closest_node(node, nodes):
    nodes = np.asarray(nodes)
    deltas = nodes - node
    dist_2 = np.einsum('ij,ij->i', deltas, deltas)
    return dist_2.argsort()

def closest(X, p):
    disp = X - p
    return np.argmin((disp * disp).sum(1))

def sample_Z(batch, z_dim, sampler='one_hot', num_class=10, n_cat=1, label_index=None, save_label=False):
    if sampler == 'mul_cat':
        if label_index is None:
            label_index = np.random.randint(low=0, high=num_class, size=batch)
        return np.hstack((0.10 * np.random.randn(batch, z_dim - num_class * n_cat),
                          np.tile(np.eye(num_class)[label_index], (1, n_cat))))
    elif sampler == 'one_hot':
        if label_index is None:
            label_index = np.random.randint(low=0, high=num_class, size=batch)
        if save_label:
            return np.hstack(
                (0.10 * np.random.randn(batch, z_dim - num_class), np.eye(num_class)[label_index])), label_index
        else:
            return np.hstack((0.10 * np.random.randn(batch, z_dim - num_class), np.eye(num_class)[label_index]))
    elif sampler == 'uniform':
        return np.random.uniform(-1., 1., size=[batch, z_dim])
    elif sampler == 'normal':
        return 0.15 * np.random.randn(batch, z_dim)
    elif sampler == 'mix_gauss':
        if label_index is None:
            label_index = np.random.randint(low=0, high=num_class, size=batch)
        return (0.1 * np.random.randn(batch, z_dim) + np.eye(num_class, z_dim)[label_index])

def sample_labelled_Z(batch, z_dim, sampler='one_hot', num_class=10, n_cat=1, label_index=None):
    if sampler == 'mul_cat':
        if label_index is None:
            label_index = np.random.randint(low=0, high=num_class, size=batch)
        return label_index, np.hstack((0.10 * np.random.randn(batch, z_dim - num_class * n_cat),
                                       np.tile(np.eye(num_class)[label_index], (1, n_cat))))
    elif sampler == 'one_hot':
        if label_index is None:
            label_index = np.random.randint(low=0, high=num_class, size=batch)
        return label_index, np.hstack(
            (0.10 * np.random.randn(batch, z_dim - num_class), np.eye(num_class)[label_index]))
    elif sampler == 'mix_gauss':
        if label_index is None:
            label_index = np.random.randint(low=0, high=num_class, size=batch)
        return label_index, (0.1 * np.random.randn(batch, z_dim) + np.eye(num_class, z_dim)[label_index])

def reshape_mnist(X):
    return X.reshape(X.shape[0], 28, 28, 1)

def clus_sample_Z(batch, dim_gen=20, dim_c=2, num_class=10, label_index=None):
    if label_index is None:
        label_index = np.random.randint(low=0, high=num_class, size=batch)
    batch_mat = np.zeros((batch, num_class * dim_c))
    for b in range(batch):
        batch_mat[b, label_index[b] * dim_c:(label_index[b] + 1) * dim_c] = np.random.normal(loc=1.0, scale=0.05,
                                                                                size=(1, dim_c))
    return np.hstack((0.10 * np.random.randn(batch, dim_gen), batch_mat))

def clus_sample_labelled_Z(batch, dim_gen=20, dim_c=2, num_class=10, label_index=None):
    if label_index is None:
        label_index = np.random.randint(low=0, high=num_class, size=batch)
    batch_mat = np.zeros((batch, num_class * dim_c))
    for b in range(batch):
        batch_mat[b, label_index[b] * dim_c:(label_index[b] + 1) * dim_c] = np.random.normal(loc=1.0, scale=0.05,
                                                                                size=(1, dim_c))
    return label_index, np.hstack((0.10 * np.random.randn(batch, dim_gen), batch_mat))

def sample_info(batch, z_dim, sampler='one_hot', num_class=10, n_cat=1, label_index=None):
    if sampler == 'one_hot':
        if label_index is None:
            label_index = np.random.randint(low=0, high=num_class, size=batch)
        return label_index, np.hstack(
            (np.random.randn(batch, z_dim - num_class), np.eye(num_class)[label_index]))
    elif sampler == 'mul_cat':
        if label_index is None:
            label_index = np.random.randint(low=0, high=num_class, size=batch)
        return label_index, np.hstack((np.random.randn(batch, z_dim - num_class * n_cat),
                                       np.tile(np.eye(num_class)[label_index], (1, n_cat))))

if __name__ == '__main__':
    load_termisk_reduced()


import os
from os import listdir
from zipfile import ZipFile
import img2pdf

#Get Current Working Directory
zipfolder = os.getcwd()

# read all zip files in folder
for zip_files in os.listdir(zipfolder):# 這三行可以用來避免'.DS_Store' problem in Mac
    if not zip_files.endswith(".zip"):
        if not zip_files.endswith(".rar"):
            continue

    clean_name = os.path.splitext(zip_files)[0]
    dirname = zipfolder+'/'+clean_name

# create a folder by zip_files's name
    if zip_files.endswith('.zip'):
        os.makedirs(clean_name)
# 把zip files的圖片unzip到上面建立的folder內
    if zip_files.endswith('.zip'):
        with ZipFile(zip_files, 'r') as zipObj:
            zipObj.extractall(dirname)

# 把PDF放在當前folder內，二選一
    with open(f"{clean_name}.pdf", "wb") as f:
        imgs = []
        for pdf_files in os.listdir(dirname):
            if not pdf_files.endswith(".jpg"):
                if not pdf_files.endswith(".png"):
                    continue
            path = os.path.join(dirname, pdf_files)
            if os.path.isdir(path):
                continue
            imgs.append(path)
            imgs.sort()
        f.write(img2pdf.convert(imgs))

# 把PDF放在圖片新開的folder內，二選一
    # pdf_path = zipfolder+'/'+'/'+clean_name+'/'+clean_name+'.pdf'
    # with open(pdf_path, "wb") as f:
    #     imgs = []
    #     for pdf_files in os.listdir(dirname):
    #         if not pdf_files.endswith(".jpg"):
    #             if not pdf_files.endswith(".png"):
    #                 continue
    #         path = os.path.join(dirname, pdf_files)
    #         if os.path.isdir(path):
    #             continue
    #         imgs.append(path)
    #         imgs.sort()
    #     f.write(img2pdf.convert(imgs))

# can using `rm .DS_Store` to detele this file first then run these code


'''
LeetCode #701 - Insert Into a Binary Search Tree prompt:

You are given the root node of a binary search tree (BST) and a
value to insert into the tree. Return the root node of the BST
after the insertion. It is guaranteed that the new value does
not exist in the original BST.

Notice that there may exist multiple valid ways for the
insertion, as long as the tree remains a BST after insertion.
You can return any of them.

Example 1:
Input: root = [4,2,7,1,3], val = 5
Output: [4,2,7,1,3,5]
Explanation: Another accepted tree is:

Example 2:
Input: root = [40,20,60,10,30,50,70], val = 25
Output: [40,20,60,10,30,50,70,null,null,25]

Example 3:
Input: root = [4,2,7,1,3,null,null,null,null,null,null], val = 5
Output: [4,2,7,1,3,5]

Constraints:
The number of nodes in the tree will be in the range [0, 10^4].
-10^8 <= Node.val <= 10^8
All the values Node.val are unique.
-10^8 <= val <= 10^8
It's guaranteed that val does not exist in the original BST.
'''
from typing import Optional
# Definition for a binary tree node.
class TreeNode:
	def __init__(self, val: int = 0, left = None, right = None):
		self.val = val
		self.left = left        # left < Parent Node
		self.right = right      # right > Parent Node

class Solution:
	# Time complexity:  O(log n) = O(h)
	# Space complexity: O(log n)
	def insertIntoBST(self, root: Optional[TreeNode], val: int) -> Optional[TreeNode]:
		if not root:    # if tree doesnt exist insert single node tree
			return TreeNode(val) # & if parent node points to NULL insert node

		if val < root.val:  # iterate left
			root.left = self.insertIntoBST(root.left, val)
		else:           # iterate right
			root.right = self.insertIntoBST(root.right, val)

		return root

# Ex 1:
# Input: root = [4,2,7,1,3], val = 5
# Output: [4,2,7,1,3,5]
root = TreeNode(4)
root.left = TreeNode(2)
root.left.left = TreeNode(1)
root.left.right = TreeNode(3)
root.right = TreeNode(7)
val = 5
result = Solution().insertIntoBST(root, val)
assert result.val == 4
assert result.left.val == 2
assert result.left.left.val == 1
assert result.left.right.val == 3
assert result.right.val == 7
assert result.right.left.val == 5

# Ex 2:
# Input: root = [40,20,60,10,30,50,70], val = 25
# Output: [40,20,60,10,30,50,70,null,null,25]
root = TreeNode(40)
root.left = TreeNode(20)
root.left.left = TreeNode(10)
root.left.right = TreeNode(30)
root.right = TreeNode(60)
root.right.left = TreeNode(50)
root.right.right = TreeNode(70)
val = 25
result = Solution().insertIntoBST(root, val)
assert result.val == 40
assert result.left.val == 20
assert result.left.left.val == 10
assert result.left.right.val == 30
assert result.right.val == 60
assert result.right.left.val == 50
assert result.right.right.val == 70
assert result.left.right.left.val == 25

# Ex 3:
# Input: root = [4,2,7,1,3,null,null,null,null,null,null], val = 5
# Output: [4,2,7,1,3,5]
root = TreeNode(4)
root.left = TreeNode(2)
root.left.left = TreeNode(1)
root.left.right = TreeNode(3)
root.right = TreeNode(7)
val = 5
result = Solution().insertIntoBST(root, val)
assert result.val == 4
assert result.left.val == 2
assert result.left.left.val == 1
assert result.left.right.val == 3
assert result.right.val == 7
assert result.right.left.val == 5

Trees/LeetCode #701 - Insert into a Binary Search Tree.py


#!/usr/bin/env python3
import pprint, argparse, pickle, json

import maze

def fail(message):
    return {
        'score'     : 0,
        'output'    : message,
        'visibility': 'visible',
    }

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description     = 'CS440 MP1 Autograder',
        formatter_class = argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--gradescope', default = False, action = 'store_true',
                        help = 'save output in gradescope-readable json file')

    arguments   = parser.parse_args()

    try:
        import search
    except ImportError:
        message = 'could not find module \'search\', did you upload `search.py`?'
        if arguments.gradescope:
            with open('results.json', 'w') as file:
                file.write(json.dumps(fail(message)))
        else:
            print(message)
        raise SystemExit

def generate_answer_key(path, mazes, solutions):
    key_instructor  = tuple({case: getattr(search, solution)(maze)
        for case, maze in mazes.items()}
        for mazes, solution in zip(mazes, solutions))
    key_student     = tuple({case: len(path) for case, path in part.items()}
        for part in key_instructor)
    pickle.dump(key_instructor, open(path['instructor'], 'wb'))
    pickle.dump(key_student,    open(path['student'],    'wb'))

def load_answer_key(path):
    try:
        return pickle.load(open(path['instructor'], 'rb'))
    except FileNotFoundError:
        print('running in student mode (instructor key unavailable)')
        return pickle.load(open(path['student'],    'rb'))

def grade_closed(name, key, mazes, solution, weight = 1):
    def grade(case, maze):
        y = key[case]
        z = getattr(search, solution)(maze)
        # check that the path is valid
        score_validity  = int(maze.validate_path(z) is None)
        # check that the length of the student’s path matches
        score_length    = int(len(z) == (y if type(y) is int else len(y)))
        # if instructor key available, check that path vertices match
        score_vertices  = int(type(y) is not int and z == y)

        return (
            {
                'name'      : '{0}: `validate_path(_:)` for \'{1}\' maze'.format(name, case),
                'score'     : 0.5 * weight * score_validity,
                'max_score' : 0.5 * weight,
                'visibility': 'visible'
            },
            {
                'name'      : '{0}: correct path length for \'{1}\' maze'.format(name, case),
                'score'     : 0.5 * weight * score_length,
                'max_score' : 0.5 * weight,
                'visibility': 'visible'
            },
            #{
            #    'name'      : '{0}: correct path vertices for \'{1}\' maze'.format(name, case),
            #    'score'     : 0.75 * weight * score_vertices,
            #    'max_score' : 0.75 * weight * (type(y) is not int),
            #    'visibility': 'visible'
            #},
        )

    return tuple(item for case, maze in mazes.items() for item in grade(case, maze))

def grade_open(name, key, mazes, solution, weight = 1):
    def grade(case, maze):
        y = key[case]
        z = getattr(search, solution)(maze)
        # check that the path is valid
        score_validity  = int(maze.validate_path(z) is None)
        # score student path by dividing the length of an MST-based solution by
        # the length of the student path
        score_length    = min((y if type(y) is int else len(y)) / max(len(z), 1), 1)

        return (
            {
                'name'      : '{0}: `validate_path(_:)` for \'{1}\' maze'.format(name, case),
                'score'     : 0.5 * weight * score_validity,
                'max_score' : 0.5 * weight,
                'visibility': 'visible'
            },
            {
                'name'      : '{0}: path length for \'{1}\' maze'.format(name, case),
                'score'     : 0.5 * weight * score_length,
                'max_score' : 0.5 * weight,
                'visibility': 'visible'
            },
        )

    return tuple(item for case, maze in mazes.items() for item in grade(case, maze))

def main():
    solutions = ('bfs', 'astar_single', 'astar_corner', 'astar_multiple', 'fast')
    for solution in solutions:
        if not hasattr(search, solution):
            return fail('module \'search\' is missing expected member \'{0}\''.format(solution))
        if not callable(getattr(search, solution)):
            return fail('member \'{0}\' in module \'search\' is not callable'.format(solution))

    mazes = (
        # part 1: 20 points total, 4 points per case
        {case: maze.maze('data/part-1/{0}'.format(case))
            for case in ('tiny', 'small', 'medium', 'large', 'open')},
        # part 2: 20 points total, 4 points per case
        {case: maze.maze('data/part-2/{0}'.format(case))
            for case in ('tiny', 'small', 'medium', 'large', 'open')},
        # part 3: 30 points total, 10 points per case
        {case: maze.maze('data/part-3/{0}'.format(case))
            for case in ('tiny', 'medium', 'large')},
        # part 4: 30 points total, 10 points per case
        {case: maze.maze('data/part-4/{0}'.format(case))
            for case in ('tiny', 'small', 'medium')},
        # part 5: 22 points total, 22 points per case
        #{case: maze.maze('data/part-5/{0}'.format(case))
        #    for case in ('large',)},
    )

    #generate_answer_key({'instructor': 'key', 'student': 'key-student'}, mazes, solutions)
    key             = load_answer_key({'instructor': 'key', 'student': 'key-student'})
    parts_closed    = tuple(item for i, points in zip(range(0, 4), (4, 4, 10, 10))
        for item in grade_closed('part-{0}'.format(i + 1), key[i], mazes[i], solutions[i],
            weight = points))
    #parts_open      = tuple(item for i, points in zip(range(4, 5), (22,))
    #    for item in grade_open(  'part-{0}'.format(i + 1), key[i], mazes[i], solutions[i],
    #        weight = points))

    # construct grade dictionary for gradescope
    return {
        'visibility': 'visible',
        'tests': parts_closed
        #'tests': parts_closed + parts_open
    }

if __name__ == "__main__":
    results     = main()
    if arguments.gradescope:
        with open('results.json', 'w') as file:
            file.write(json.dumps(results))
    else:
        pprint.pp(results)


import subprocess
from itertools import product, combinations

import numpy as np

def runner():
    SEQ = 'MQYKLILNGKTLKGETTTEAVDAATAEKVFKQYANDNGVDGEWTYDDATKTFTVTE'
    AA = ['A', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'L',
          'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'Y']
    AA_dict = dict((k,v) for v,k in enumerate(AA))
    SEQ = list(map(AA_dict.get, list(SEQ)))

    # Test
    mutations = []
    for pos1 in range(1):
        for mut1 in range(4):
            if mut1!=SEQ[pos1]:
                mutations.append([pos1, mut1, -1, -1])
    mutations = np.array(mutations)

    np.savetxt("./assets/mutation_input/test.csv", mutations, delimiter=",")
    run_process("test")

    # Single mutation
    mutations = []
    for pos1 in range(1,len(SEQ)):
        for mut1 in range(len(AA)):
            if mut1!=SEQ[pos1]:
                mutations.append([pos1, mut1, -1, -1])
    mutations = np.array(mutations)
    fnm = 'single_mutations'
    np.savetxt('./assets/mutation_input/'+fnm+'.csv', mutations, delimiter=',')
    run_process(fnm)

    # Double mutation
    residues_list = combinations(list(range(1,len(SEQ))),2)
    for cnt, residues in enumerate(residues_list):
        mutations = []
        for mut1 in range(20):
            for mut2 in range(20):
                if mut1!=SEQ[residues[0]] and mut2!=SEQ[residues[1]]:
                    mutations.append([residues[0],mut1,residues[1],mut2])
        mutations = np.array(mutations)
        fnm = "double_mutations_{0:d}_{1:d}".format(residues[0],residues[1])
        np.savetxt('./assets/mutation_input/'+fnm+'.csv', mutations, delimiter=',')
        run_process(fnm)
        write_files_to_run(fnm, cnt)

def write_files_to_run(fnm, cnt):
    fnm_list = ['run_files_{0:d}'.format(i) for i in range(1,4)]
    index = cnt//495
    with open(fnm_list[index], 'a+') as f:
        f.write('sbatch ./assets/sbatch_scripts/'+fnm+'.sh \n')

def run_process(fnm):
    with open('./assets/sbatch_scripts/'+fnm+'.sh', 'w+') as f:
        partitions = ['sched_mit_arupc_long',
                    'sched_mit_arupc',
                    'sched_any',
                    'sched_mit_hill',
                    'newnodes']
        f.write('#!/bin/bash \n')
        f.write('#SBATCH --job-name='+fnm+' \n')
        f.write('#SBATCH --nodes=1 \n')
        f.write('#SBATCH --cpus-per-task=2 \n')
        f.write('#SBATCH --time=12:00:00 \n')
        f.write('#SBATCH --partition='+','.join(partitions)+' \n')
        f.write('#SBATCH --mem-per-cpu=2000 \n')
        f.write('#SBATCH -o /nobackup1c/users/leerang/UniRep/output/output_%j.txt \n')
        f.write('#SBATCH -e /nobackup1c/users/leerang/UniRep/error/error_%j.txt \n\n')
        f.write('cd /nobackup1c/users/leerang/UniRep')
        f.write('module add /home/software/modulefiles/singularity/3.7.0 \n')
        f.write('singularity exec -B /nobackup1c/users/leerang/UniRep ./UniRep.sif python3 embedding_getter.py '+fnm)

if __name__ == '__main__':
    runner()
    #result = subprocess.run(["python", "embedding_getter.py", "test.csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    #print(result.stdout)
    #print(result.stderr)
    #print(result.returncode)


import socket
import os
import multiprocessing

def send_message(s_socket, client_address):
    message = 'Hi ' + client_address[0] + ':' + str(client_address[1]) + '. This is server ' + str(
        os.getpid())
    s_socket.sendto(str.encode(message), client_address)
    print('Sent to client: ', message)

def test(x):
    return x * x

if __name__ == "__main__":

    number_processes = 4
    # Initialise the pool
    pool = multiprocessing.Pool(number_processes)

    server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    # Bind the socket to the port
    server_address = '127.0.0.1'
    server_port = 10001

    # Buffer size
    buffer_size = 1024

    server_socket.bind((server_address, server_port))
    print('Server up and running at {}:{}'.format(server_address, server_port))

    while True:
        data, address = server_socket.recvfrom(buffer_size)
        print('Received message \'{}\' at {}:{}'.format(data.decode(), address[0], address[1]))
        pool.apply_async(send_message, args=(server_socket, address,))


from PIL import Image

im1 = Image.open("lena.png")
im2 = Image.open("lena_modified.png")
#im.show()

width, height = im1.size
print(width)
print(height)

for y in range(height):
    for x in range(width):
        rgba1 = im1.getpixel( (x, y) )
        rgba2 = im2.getpixel( (x, y) )
        #print(rgba)
        if rgba1 == rgba2 :
            rgba2 = (rgba2[0] - rgba1[0],  # R
                     rgba2[1] - rgba1[1],  # G
                     rgba2[2] - rgba1[2],  # B
                     rgba2[3] - rgba1[3]);  # A
        im1.putpixel((x, y), rgba2)

im1.show()
im1.save("new.png")


from rest_framework import serializers
from . import twitterhelper
from .register import register_social_user
import os
from rest_framework.exceptions import AuthenticationFailed

class TwitterAuthSerializer(serializers.Serializer):
    """Handles serialization of twitter related data"""
    access_token_key = serializers.CharField()
    access_token_secret = serializers.CharField()

    def validate(self, attrs):

        access_token_key = attrs.get('access_token_key')
        access_token_secret = attrs.get('access_token_secret')

        user_info = twitterhelper.TwitterAuthTokenVerification.validate_twitter_auth_tokens(
            access_token_key, access_token_secret)

        try:
            user_id = user_info['id_str']
            email = user_info['email']
            name = user_info['name']
            provider = 'twitter'
            first_token = attrs.get('access_token_key')
            second_toekn = attrs.get('access_token_secret')
        except:
            raise serializers.ValidationError(
                'The tokens are invalid or expired. Please login again.'
            )

        return register_social_user(
            provider=provider, user_id=user_id, email=email, name=name, first_token = first_token, second_token = second_token)


import transformers
from utils import printf
import copy

class prompt:
    def __init__(self, tokenizer, max_len, add_eos=True):
        self.tokenizer = tokenizer
        self.max_len = max_len
        self.add_eos=add_eos

class instruct_prompt(prompt):
    prompt = (
        "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Response:"
    )
    prompt_input = (
        "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n"
        "### Instruction:{instruction}\n\n### Input:{input}\n\n### Response:"
    )
    prompt_history = "User:{input}\n\nAssistant:{output}\n\n"
    prompt_post = "User:{input}\n\nAssistant:"

    def preprocess_gen(self, data_point):
        if 'history' not in data_point:
        # single instruction format {'instruction':..,'input':..}
            if 'input' in data_point:
                user_prompt = self.prompt_input.format_map(data_point)
            else:
                user_prompt = self.prompt.format_map(data_point)
        else:
        # multi turn format {'history':[..], 'input':[..]}
            user_prompt = "\n".join(["User:" + i['input']+"\n"+"Assistant:" + i['output'] for i in data_point['history']]) + "\nUser:" + data_point['input'] + "\nAssistant:"
            user_prompt = user_prompt[-self.max_len:]
        user_prompt=self.prompt.format_map({'instruction':user_prompt})
        input_ids = self.tokenizer(user_prompt)["input_ids"]
        return input_ids

    def preprocess_train(self, data_point):
        # single instruction format {'instruction':..,'input':..,'output':..}
        if 'instruction' in data_point:
            if 'input' in data_point:
                user_prompt = self.prompt_input.format_map(data_point)
            else:
                user_prompt = self.prompt.format_map(data_point)
            output = data_point["output"]
        # multi turn format {'input':[..], 'output':[..]}
        else:
            user_prompt = ''
            lens = len(data_point['input'])
            for i in range(lens-1):
                user_prompt += self.prompt_history.format_map({'input':data_point['input'][i],'output':data_point['output'][i]})
            user_prompt += self.prompt_post.format_map({'input':data_point['input'][-1]})
            user_prompt = self.prompt.format_map({'instruction': user_prompt})
            output = data_point['output'][-1]

        len_user_prompt_tokens = (len(self.tokenizer(
            user_prompt,
            truncation=True,
            max_length=self.max_len + 1,
        )["input_ids"])- 1)  # no eos token
        full_tokens = self.tokenizer(
            user_prompt + output,
            truncation=True,
            max_length=self.max_len + 1,
            padding="max_length",
        )["input_ids"][:-1]
        return {
            "input_ids": full_tokens,
            "labels": [-100] * len_user_prompt_tokens
            + full_tokens[len_user_prompt_tokens:],
            "attention_mask": [1] * (len(full_tokens)),
        }

    def data_collator(self,):
        return transformers.DataCollatorForLanguageModeling(self.tokenizer, mlm=False)

    def postprocess(self, text, render=True):
        #import pdb;pdb.set_trace()
        printf(text)
        output = text.split("### Response:")[1].strip()
        output = output.replace("Belle", "Vicuna")
        printf(output)
        if '###' in output:
            output = output.split("###")[0]
        if 'User' in output:
            output = output.split("User")[0]
        output = output.replace('�','').replace('</s>', '')
        if render:
            # fix gradio chatbot markdown code render bug
            lines = output.split("\n")
            for i, line in enumerate(lines):
                if "```" in line:
                    if line != "```":
                        lines[i] = f'
'
                    else:
                        lines[i] = '
'
                else:
                    if i > 0:
                        lines[i] = "
" + line.replace("<", "<").replace(">", ">").replace("__", '\_\_')
            output =  "".join(lines)
            # output = output.replace('

','\n
') work for html; but not for gradio
        return output

class chat_prompt(prompt):
    prompt_pre = (
        "The following is a conversation between an AI assistant called Assistant and a human user called User. "
        "The assistant is intelligent, knowledgeable and polite to answer questions of user.\n\n"
    )
    prompt_history = "User:{input}\n\nAssistant:{output}\n\n"
    prompt_post = "User:{input}\n\nAssistant:"

    def preprocess_gen(self, data_point):
        user_prompt = self.prompt_pre
        len_avail = self.max_len - len(self.tokenizer(user_prompt, add_special_tokens=False)['input_ids'])
        input_prompt = self.prompt_post.format_map({'input':data_point['input']})
        len_avail -= len(self.tokenizer(input_prompt, add_special_tokens=False)['input_ids'])
        lens = len(data_point['history'])
        tokenized_lens = []
        for i in range(lens):
            tmp_prompt = self.prompt_history.format_map(data_point['history'][i])
            tokenized_lens.append(len(self.tokenizer(tmp_prompt,add_special_tokens=False)["input_ids"]))

        # 启发式：/2 优先除前面的
        i = 0
        while sum(tokenized_lens) > len_avail and i < lens:
            history = data_point['history'][i]
            tmp_len1 = len(history['input'])
            tmp_len2 = len(history['output'])
            if tmp_len2 > tmp_len1:
                history['output'] = history['output'][:tmp_len2//2]
            else:
                history['input'] = history['input'][:tmp_len1//2]
            prompt = self.prompt_history.format_map(history)
            single_len =(len(self.tokenizer(prompt,add_special_tokens=False)["input_ids"]))
            tokenized_lens[i] = single_len
            i += 1
        total_len = sum(tokenized_lens)
        # 还不够的话 直接截断
        while total_len > len_avail and i < lens - 1 :
            total_len -= tokenized_lens[i]
            data_point['history'] = data_point['history'][1:]
            i += 1
        # 最终合并
        for i in range(lens):
            user_prompt += self.prompt_history.format_map(data_point['history'][i])
        user_prompt += input_prompt
        printf({'real_input:':user_prompt})
        inputs = self.tokenizer(user_prompt)["input_ids"]
        return inputs

    def preprocess_train(self, data_point):
        user_prompt = self.prompt_pre
        lens = len(data_point['input'])
        for i in range(lens-1):
            user_prompt += self.prompt_history.format_map({'input':data_point['input'][i].strip(),'output':data_point['output'][i].strip()})
        user_prompt += self.prompt_post.format_map({'input':data_point['input'][-1].strip()})

        len_user_prompt_tokens = len(self.tokenizer(
            user_prompt,
            truncation=True,
            max_length=self.max_len,
        )["input_ids"]) - 1 # remove extra eos
        if self.add_eos:
            full_tokens = self.tokenizer(
                user_prompt + data_point["output"][-1].strip(),
                truncation=True,
                padding=False,
                max_length=self.max_len,
            )["input_ids"] # need eos
        else:
            full_tokens = self.tokenizer(
                user_prompt + data_point["output"][-1].strip(),
                truncation=True,
                padding=False,
                max_length=self.max_len+1,
            )["input_ids"][:-1] # delete eos
        return {
            "input_ids": full_tokens,
            "labels": [-100] * len_user_prompt_tokens + full_tokens[len_user_prompt_tokens:],
            "attention_mask": [1] * (len(full_tokens)),
        }

    def data_collator(self,):
        return transformers.DataCollatorForSeq2Seq(self.tokenizer)

    def postprocess(self, text, render=False):
        output = text.split("Assistant:")[-1].strip()
        if 'User:' in output:
            output = output.split("User:")[0]
        output = output.replace('�','')
        if render:
            # fix gradio chatbot markdown code render bug
            lines = output.split("\n")
            for i, line in enumerate(lines):
                if "```" in line:
                    if line != "```":
                        lines[i] = f'
'
                    else:
                        lines[i] = '
'
                else:
                    if i > 0:
                        lines[i] = "
" + line.replace("<", "<").replace(">", ">").replace("__", '\_\_')
            output =  "".join(lines)
            # output = output.replace('

','\n
') work for html; but not for gradio
        return output

    def get_data_collator():
        return transformers.DataCollatorForLanguageModeling


from dotenv import load_dotenv
from logging import error
import telegram
import requests
import os
import logging
from telegram.ext import CommandHandler, Updater, Filters, MessageHandler
from telegram.message import Message
from pprint import pprint

load_dotenv()

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s, %(levelname)s, %(message)s, %(name)s'
)

TOKEN = os.getenv('TOKEN')
CHAT_ID = os.getenv('CHAT_ID')

bot = telegram.Bot(token=TOKEN)
updater = Updater(token=TOKEN)

URL = 'https://api.thecatapi.com/v1/images/search'

def start_work(update, context):
    chat = update.effective_chat
    name = update.message.chat.first_name
    button = telegram.ReplyKeyboardMarkup(
        [
            ['/newcat'],
            ['Выход']
        ], resize_keyboard=True
    )
    context.bot.send_message(
        chat_id=chat.id,
        text='спс, от души, брат-{}!'.format(name),
        reply_markup=button
    )
    context.bot.send_photo(chat.id, get_image())

def get_image():
    try:
        response = requests.get(URL)
    except Exception:
        logging.exception(f'Ошибка при запросе к основному API: {error}')
        NEW_URL = 'https://api.thedogapi.com/v1/images/search'
        response = requests.get(NEW_URL)
    rnd_cat = response.json()[0]['url']
    return rnd_cat

def put_image(update, context):
    chat = update.effective_chat
    context.bot.send_photo(chat.id, get_image())

def say_hi(update, context):
    chat = update.effective_chat
    context.bot.send_message(chat_id=chat.id, text='message')

def main():
    updater.dispatcher.add_handler(CommandHandler('start', start_work))
    updater.dispatcher.add_handler(CommandHandler('newcat', put_image))
    updater.dispatcher.add_handler(MessageHandler(Filters.text, say_hi))
    updater.start_polling(poll_interval=3.0)
    updater.idle()

if __name__ == '__main__':
    main()

def send_message(message):
    bot.send_message(chat_id=CHAT_ID, text=message)

text = 'Вам телеграмма!'

send_message(text)


import xml.etree.ElementTree as ET
import subprocess
import asyncio
import sys

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

async def execute11(line):
    cmd = "openssl s_client -connect %s < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin" % line
    fp_ = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
    stdout, stderr = await fp_.communicate()
    f[line] = stdout
    return f

print(f"""{bcolors.HEADER}
 _____ _           _ ____                        ____          _
 |  ___(_)_ __   __| / ___|  __ _ _ __ ___   ___ / ___|___ _ __| |_
 | |_  | | '_ \ / _` \___ \ / _` | '_ ` _ \ / _ \ |   / _ \ '__| __|
 |  _| | | | | | (_| |___) | (_| | | | | | |  __/ |__|  __/ |  | |_
 |_|   |_|_| |_|\__,_|____/ \__,_|_| |_| |_|\___|\____\___|_|   \__|
{bcolors.ENDC}"""
)

argc = len(sys.argv)
if argc == 1 or argc == 2:
	print('Usage:\n\tpython3 FindSameCert.py domain:port nmap.xml')
	print('For example:\n\tpython3 FindSameCert.py kkk.com:443 nmap.xml')
	sys.exit()
else:
	url = sys.argv[1]
	nmap = sys.argv[2]

tree = ET.parse(nmap)
root = tree.getroot()

SslPorts = []
ListDomCheck = {}
hosts = root.findall('host')

for host in hosts:
    try:
        HostNameElem = host.findall('hostnames')
        HostName = HostNameElem[0].findall('hostname')[0].attrib['name']
        PortElement = host.findall('ports')
        ports = PortElement[0].findall('port')
        for port in ports:
            service = port.findall('service')[0].attrib['name']
            if "tunnel" in port.findall('service')[0].attrib:
                _port = port.attrib['portid']
                SslPorts.append(_port)
            SslPortsCopy = SslPorts.copy()
            ListDomCheck[HostName] = SslPortsCopy
        SslPorts.clear()
    except:
        pass

DomainList = []
for i in ListDomCheck:
    for j in range(len(ListDomCheck[i])):
        line = i+":"+ListDomCheck[i][j]
        DomainList.append(line)

f = {}

cmd = "openssl s_client -connect %s < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin" %url
fp = subprocess.Popen([cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()[0]
print (f"{bcolors.HEADER}TLS certificate fingerprint for %s:{bcolors.ENDC}" %url)
print(fp.decode("utf-8"))

futures = [execute11(domain) for domain in DomainList]

loop = asyncio.get_event_loop()
fp_ = loop.run_until_complete(asyncio.wait(futures))

print(f"{bcolors.HEADER}The same TLS certificate is used on:{bcolors.ENDC}")
for k, v in f.items():
    if v == fp:
        print(f"{bcolors.BOLD}%s{bcolors.ENDC}" %k)


from flask import request
from flask.json import jsonify
from . import habits
from .models import Habit
from .dbmodel import Habitdb
import json
from bson import json_util
from flask_jwt_extended import get_jwt_identity,jwt_required

@habits.route('/',methods=['POST'])
@jwt_required()
def create():
    current_user = get_jwt_identity()
    habit = Habit(**request.json)
    new_habit=Habitdb().create_habit(habit,current_user)
    if(new_habit):
        return jsonify(request.json),200
    return 'Error',409

@habits.route('/',methods=['GET'])
@jwt_required()
def get():
    current_user = get_jwt_identity()
    habit_db=Habitdb().get_all_habits(current_user)
    response=json.loads(json_util.dumps(habit_db))
    return jsonify(response),200

@habits.route('/<string:_id>',methods=['DELETE'])
@jwt_required()
def delete(_id):
    habit_db=Habitdb().delete_habit(_id)
    if(habit_db):
        return 'Sucess',200
    return 'Cant delete it',409

@habits.route('/<string:_id>',methods=['PUT'])
@jwt_required()
def edit(_id):
    habit=Habitdb().get_habit_Object(_id)
    habit.changes(**request.json)
    habit_db=Habitdb().update_habit(habit)
    if(habit_db):
        return jsonify(request.json),200
    return 'Cant edit anything',405

@habits.route('/today/<string:_id>',methods=['POST'])
@jwt_required()
def marktoday(_id):
    r=request.json
    habit=Habitdb().get_habit_Object(_id)
    try:
        if(r['mark']):
            habit.date.mark_today()
        else:
            habit.date.unmark_today()
        update_habit=Habitdb().update_habit(habit)
        if(update_habit):
            return 'Sucess',200
    except KeyError:
        return 'error in date',405


from pydantic import ValidationError

from saltapi.web.schema.common import ProposalCode
import pytest

@pytest.mark.parametrize("code", ["2021-1-SCI-001", "2022-2-ORP-007", "2023-1-DDT-001"])
def test_valid_proposal_code(code: str):
    assert ProposalCode.validate(code) == code

@pytest.mark.parametrize(
    "invalid_code",
    [
        "20a1-1-ABC-123",  # Invalid year
        "2021-3-ABC-123",  # Invalid semester
        "2021-1-_ABC-123",  # Underscore can't start the letter sequence
        "2021-1-ABC_-123",  # Underscore can't end the letter sequence
        "2021-1-ABC-12",  # Invalid number of digits
    ],
)
def test_invalid_proposal_code(invalid_code: str):
    with pytest.raises(ValueError, match="incorrect proposal code"):
        ProposalCode.validate(invalid_code)

def test_non_string_input():
    with pytest.raises(TypeError, match="string required"):
        ProposalCode.validate(123)