import itertools
import math

import numpy as np

class CRF(object):
    def __init__(self, units, transitions, transitions_from_start, transitions_to_end, transition_constraint_mask=None, dynamic_transition_constraint=None):
        self.units = units
        self.transitions = transitions
        self.transitions_from_start = transitions_from_start
        self.transitions_to_end = transitions_to_end
        self.transition_constraint_mask = transition_constraint_mask
        self.dynamic_transition_constraint = dynamic_transition_constraint

    def score(self, logits, tags, transitions_from_start=None, transitions_to_end=None, transitions=None):
        """
        Computes the likelihood score for the given sequence of tags,
        given the provided logits (and the transition weights in the CRF model)
        """
        if transitions_from_start is None:
            transitions_from_start = self.transitions_from_start

        if transitions_to_end is None:
            transitions_to_end = self.transitions_to_end

        if transitions is None:
            transitions = self.transitions

        # Start with transitions from START and to END
        total = transitions_from_start[tags[0]] + transitions_to_end[tags[-1]]
        # Add in all the intermediate transitions
        for tag, next_tag in zip(tags, tags[1:]):
            total += transitions[tag, next_tag]
        # Add in the logits for the observed tags
        for logit, tag in zip(logits, tags):
            total += logit[tag]
        return total

    def compute_log_likelihood_without_mask(self, logits, tags):
        # Now compute the log-likelihood manually
        manual_log_likelihood = 0.0

        # For each instance, manually compute the numerator
        # (which is just the score for the logits and actual tags)
        # and the denominator
        # (which is the log-sum-exp of the scores for the logits across all possible tags)
        for logits_i, tags_i in zip(logits, tags):
            numerator = self.score(logits_i, tags_i)
            all_scores = [self.score(logits_i, tags_j)
                          for tags_j in itertools.product(range(5), repeat=3)]
            denominator = math.log(sum(math.exp(score) for score in all_scores))
            # And include them in the manual calculation.
            manual_log_likelihood += numerator - denominator

        return manual_log_likelihood

    def compute_log_likelihood_with_mask(self, logits, tags, mask):
        # Now compute the log-likelihood manually
        manual_log_likelihood = 0.0

        # For each instance, manually compute the numerator
        #   (which is just the score for the logits and actual tags)
        # and the denominator
        #   (which is the log-sum-exp of the scores for the logits across all possible tags)
        for logits_i, tags_i, mask_i in zip(logits, tags, mask):
            # Find the sequence length for this input and only look at that much of each sequence.
            sequence_length = np.sum(mask_i)
            logits_i = logits_i[:sequence_length]
            tags_i = tags_i[:sequence_length]

            numerator = self.score(logits_i, tags_i)
            all_scores = [self.score(logits_i, tags_j)
                          for tags_j in itertools.product(range(5), repeat=sequence_length)]
            denominator = math.log(sum(math.exp(score) for score in all_scores))
            # And include them in the manual calculation.
            manual_log_likelihood += numerator - denominator

        return manual_log_likelihood

    def compute_effective_boundary(self):
        """

        :return: Tensor(shape=(B, 1, n)) or Tensor(shape=(1, 1, n))
        """
        left_boundary = self.transitions_from_start
        right_boundary = self.transitions_to_end

        if self.transition_constraint_mask is not None:
            start_tag = self.units
            end_tag = self.units + 1

            left_boundary = (
                    self.transitions_from_start * self.transition_constraint_mask[start_tag, :self.units] +
                    -10000.0 * (1 - self.transition_constraint_mask[start_tag, :self.units])
            )
            right_boundary = (
                    self.transitions_to_end * self.transition_constraint_mask[self.units, end_tag] +
                    -10000.0 * (1 - self.transition_constraint_mask[self.units, end_tag])
            )

        if self.dynamic_transition_constraint is not None:
            start_tag = self.units
            end_tag = self.units + 1

            # shape: (B, 1, n)
            left_dynamic_constraint = self.dynamic_transition_constraint[:, start_tag, :self.units]

            # shape: (B, 1, n)
            left_boundary = (
                self.transitions_from_start * left_dynamic_constraint +
                -10000.0 * (1 - left_dynamic_constraint)
            )

            # shape: (B, 1, n)
            right_dynamic_constraint = self.dynamic_transition_constraint[:, :self.units, end_tag]

            # shape: (B, 1, n)
            right_boundary = (
                self.transitions_to_end * right_dynamic_constraint +
                -10000.0 * (1 - right_dynamic_constraint)
            )

        return left_boundary, right_boundary

    def compute_effective_chain_kernel(self):
        """

        :return: Tensor(shape=(B, n, n)) or Tensor(shape=(n, n))
        """
        chain_kernel = self.transitions

        if self.transition_constraint_mask is not None:
            chain_kernel_mask = self.transition_constraint_mask[:self.units, :self.units]
            chain_kernel = self.transitions * chain_kernel_mask + -10000.0 * (1 - chain_kernel_mask)

        if self.dynamic_transition_constraint is not None:
            dynamic_chain_kernel_mask = self.dynamic_transition_constraint[:, :self.units, :self.units]
            chain_kernel = self.transitions * dynamic_chain_kernel_mask + -10000.0 * (1 - dynamic_chain_kernel_mask)

        return chain_kernel

    def viterbi_decode(self, logits, mask=None):
        if mask is None:
            mask = np.ones(np.shape(logits)[:2], dtype=np.bool)

        transitions_from_start, transitions_to_end = self.compute_effective_boundary()
        transitions = self.compute_effective_chain_kernel()

        most_likely_tags = []
        best_scores = []

        for logit, mas, from_start, to_end, transition in zip(logits, mask, transitions_from_start, transitions_to_end, transitions):
            sequence_length = np.sum(mas)
            most_likely, most_likelihood = None, -float('inf')
            for tags in itertools.product(range(5), repeat=sequence_length):
                score = self.score(logit, tags, from_start, to_end, transition)
                if score > most_likelihood:
                    # padding tags to sequence length
                    tag_len_diff = 3 - len(tags)
                    if tag_len_diff:
                        tags = list(tags) + [0] * tag_len_diff

                    most_likely, most_likelihood = tags, score
            # Convert tuple to list; otherwise == complains.
            most_likely_tags.append(list(most_likely))
            best_scores.append(most_likelihood)

        return most_likely_tags


import apache_beam as beam
from apache_beam.io.gcp.internal.clients import bigquery
from apache_beam.options.pipeline_options import PipelineOptions
import os
os.environ['GOOGLE_APPLICATION_CREDENTIALS']='D:\Pycharm\GCP\key-bigquery-practice-sample-fb58a943c40b.json'

import argparse

def parseArgs():
  parser = argparse.ArgumentParser()
  parser.add_argument(
    '--experiment',
    default='use_unsupported_python_version',
    help='This does not seem to do anything.')
  args, beam_args = parser.parse_known_args()
  return beam_args

def beamer(rows=[]):
  if len(rows) == 0:
    return

  project = 'bigquery-practice-sample'
  gcs_temp_location = 'gs://gcs-sample-bucket/dataflow-error-bucket'
  gcs_staging_location = 'gs://gcs-sample-bucket/'

  table_spec = bigquery.TableReference(
    projectId=project,
    datasetId='gcp_practice_sample',
    tableId='emp_sample_details_dataflow')
  beam_options = PipelineOptions(
    parseArgs(), # This doesn't seem to work.
    project=project,
    runner='DataflowRunner',
    job_name='sample-run-pipeline',
    temp_location=gcs_temp_location,
    staging_location=gcs_staging_location,
    region='us-east1',
    use_unsupported_python_version=True, # This doesn't work either. :(
    experiments=['use_unsupported_python_version'] # This also doesn't work.
  )

  with beam.Pipeline(options=beam_options) as p:
    quotes = p | beam.Create(rows)

    quotes | beam.io.WriteToBigQuery(
    table_spec,
    # custom_gcs_temp_location = gcs_temp_location, # Not needed?
    method='FILE_LOADS',
    schema={
      "fields":[{"name":"Name","type":"STRING","mode":"REQUIRED"},
                      {"name":"EmpId","type":"INTEGER","mode":"REQUIRED"},
                      {"name": "Salary", "type":"INTEGER","mode": "REQUIRED"},
                      {"name": "Dept", "type":"String","mode": "REQUIRED"},
                      {"name": "DeptId", "type":"INTEGER", "mode": "REQUIRED"},
                      {"name": "Designation", "type":"String", "mode": "REQUIRED"},
                                            ]},
    write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
    create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED)
  return

if __name__ == '__main__':
    beamer(rows=[{'Name': 'Mohan','EmpId':345,'Salary':657,'Dept':'Finance','DeptId':11,'Designation':'Accountant'}])


# -*-coding: utf-8 -*-

#金額と個数の入力
sum,num = map(int,input().split())
#嫌いな数字の入力
dis = list(map(int,input().split()))

pay = sum #支払い金額
#1ずつ支払い金額を足して嫌いな文字が含まれていないか確認
while True:
  flag = 0 #嫌いな文字が含まれているか
  str_pay = str(pay)
  for i in str_pay:
    if int(i) in dis:
      flag = 1
      break
  if flag == 0:
    break
  else:
    pay = pay + 1
print(pay)


import re
import requests
import base64
import os
from pathlib import Path

from .peasLoaded import PEASLoaded
from .peassRecord import PEASRecord
from .fileRecord import FileRecord
from .yamlGlobals import (
    TEMPORARY_LINPEAS_BASE_PATH,
    PEAS_FINDS_MARKUP,
    PEAS_FINDS_CUSTOM_MARKUP,
    PEAS_STORAGES_MARKUP,
    INT_HIDDEN_FILES_MARKUP,
    ROOT_FOLDER,
    STORAGE_TEMPLATE,
    FIND_TEMPLATE,
    FIND_LINE_MARKUP,
    STORAGE_LINE_MARKUP,
    STORAGE_LINE_EXTRA_MARKUP,
    EXTRASECTIONS_MARKUP,
    PEAS_VARIABLES_MARKUP,
    YAML_VARIABLES,
    SUIDVB1_MARKUP,
    SUIDVB2_MARKUP,
    SUDOVB1_MARKUP,
    SUDOVB2_MARKUP,
    CAP_SETUID_MARKUP,
    CAP_SETGID_MARKUP,
    LES_MARKUP,
    LES2_MARKUP,
    REGEXES_LOADED,
    REGEXES_MARKUP,
    FAT_LINPEAS_AMICONTAINED_MARKUP,
    FAT_LINPEAS_GITLEAKS_LINUX_MARKUP,
    FAT_LINPEAS_GITLEAKS_MACOS_MARKUP
)

class LinpeasBuilder:
    def __init__(self, ploaded:PEASLoaded):
        self.ploaded = ploaded
        self.hidden_files = set()
        self.bash_find_f_vars, self.bash_find_d_vars = set(), set()
        self.bash_storages = set()
        self.__get_files_to_search()
        with open(TEMPORARY_LINPEAS_BASE_PATH, 'r') as file:
            self.linpeas_sh = file.read()

    def build(self):
        print("[+] Building variables...")
        variables = self.__generate_variables()
        self.__replace_mark(PEAS_VARIABLES_MARKUP, variables, "")

        print("[+] Building finds...")
        find_calls, find_custom_calls = self.__generate_finds()
        self.__replace_mark(PEAS_FINDS_MARKUP, find_calls, "  ")
        self.__replace_mark(PEAS_FINDS_CUSTOM_MARKUP, find_custom_calls, "  ")

        print("[+] Building storages...")
        storage_vars = self.__generate_storages()
        self.__replace_mark(PEAS_STORAGES_MARKUP, storage_vars, "  ")

        #Check all the expected STORAGES in linpeas have been created
        for s in re.findall(r'PSTORAGE_[\w]*', self.linpeas_sh):
            assert s in self.bash_storages, f"{s} isn't created"

        #Replace interesting hidden files markup for a list of all the searched hidden files
        self.__replace_mark(INT_HIDDEN_FILES_MARKUP, sorted(self.hidden_files), "|")

        print("[+] Checking duplicates...")
        peass_marks = self.__get_peass_marks()
        for i,mark in enumerate(peass_marks):
            for j in range(i+1,len(peass_marks)):
                assert mark != peass_marks[j], f"Found repeated peass mark: {mark}"

        print("[+] Building autocheck sections...")
        sections = self.__generate_sections()
        for section_name, bash_lines in sections.items():
            mark = "peass{"+section_name+"}"
            if mark in peass_marks:
                self.__replace_mark(mark, list(bash_lines), "")
            else:
                self.__replace_mark(EXTRASECTIONS_MARKUP, [bash_lines, EXTRASECTIONS_MARKUP], "\n\n")

        self.__replace_mark(EXTRASECTIONS_MARKUP, list(""), "") #Delete extra markup

        print("[+] Building regexes searches...")
        section = self.__generate_regexes_search()
        self.__replace_mark(REGEXES_MARKUP, list(section), "")

        print("[+] Building linux exploit suggesters...")
        les_b64, les2_b64 = self.__get_linux_exploit_suggesters()
        assert len(les_b64) > 100
        assert len(les2_b64) > 100
        self.__replace_mark(LES_MARKUP, list(les_b64), "")
        self.__replace_mark(LES2_MARKUP, list(les2_b64), "")

        print("[+] Downloading Fat Linpeas binaries...")
        aimcont_b64 = self.__get_bin("https://github.com/genuinetools/amicontained/releases/latest/download/amicontained-linux-amd64")
        self.__replace_mark(FAT_LINPEAS_AMICONTAINED_MARKUP, list(aimcont_b64), "")

        gitleaks_b64 = self.__get_bin("https://github.com/zricethezav/gitleaks/releases/download/v8.8.7/gitleaks_8.8.7_linux_x64.tar.gz", tar_gz="gitleaks")
        self.__replace_mark(FAT_LINPEAS_GITLEAKS_LINUX_MARKUP, list(gitleaks_b64), "")

        gitleaks_b64_macos = self.__get_bin("https://github.com/zricethezav/gitleaks/releases/download/v8.8.7/gitleaks_8.8.7_darwin_x64.tar.gz", tar_gz="gitleaks")
        self.__replace_mark(FAT_LINPEAS_GITLEAKS_MACOS_MARKUP, list(gitleaks_b64_macos), "")

        print("[+] Building GTFOBins lists...")
        suidVB, sudoVB, capsVB = self.__get_gtfobins_lists()
        assert len(suidVB) > 185, f"Len suidVB is {len(suidVB)}"
        assert len(sudoVB) > 250, f"Len sudo is {len(sudoVB)}"
        assert len(capsVB) > 10, f"Len suidVB is {len(capsVB)}"

        self.__replace_mark(SUIDVB1_MARKUP, suidVB[:int(len(suidVB)/2)], "|")
        self.__replace_mark(SUIDVB2_MARKUP, suidVB[int(len(suidVB)/2):], "|")
        self.__replace_mark(SUDOVB1_MARKUP, sudoVB[:int(len(sudoVB)/2)], "|")
        self.__replace_mark(SUDOVB2_MARKUP, sudoVB[int(len(sudoVB)/2):], "|")
        self.__replace_mark(CAP_SETUID_MARKUP, capsVB, "|")
        self.__replace_mark(CAP_SETGID_MARKUP, capsVB, "|")

        print("[+] Final sanity checks...")
        #Check that there arent peass marks left in linpeas
        peass_marks = self.__get_peass_marks()
        assert len(peass_marks) == 0, f"There are peass marks left: {', '.join(peass_marks)}"

        #Check for empty seds
        assert 'sed -${E} "s,,' not in self.linpeas_sh

    def __get_peass_marks(self):
        return re.findall(r'peass\{[\w\-\._ ]*\}', self.linpeas_sh)

    def __generate_variables(self):
        """Generate the variables from the yaml to set into linpeas bash script"""
        variables_bash = ""
        for var in YAML_VARIABLES:
            variables_bash += f"{var['name']}=\"{var['value']}\"\n"

        return variables_bash

    def __get_files_to_search(self):
        """Given a PEASLoaded and find the files that need to be searched on each root folder"""
        self.dict_to_search = {"d": {}, "f": {}}
        self.dict_to_search["d"] = {r: set() for r in ROOT_FOLDER}
        self.dict_to_search["f"] = {r: set() for r in ROOT_FOLDER}

        for precord in self.ploaded.peasrecords:
            for frecord in precord.filerecords:
                for folder in frecord.search_in:
                    self.dict_to_search[frecord.type][folder].add(frecord.regex)

                if frecord.regex[0] == "." or frecord.regex[:2] == "*.":
                    self.hidden_files.add(frecord.regex.replace("*",""))

    def __generate_finds(self) -> list:
        """Given the regexes to search on each root folder, generate the find command"""
        finds = []

        finds_custom = []
        all_folder_regexes = []
        all_file_regexes = []

        for type,searches in self.dict_to_search.items():
            for r,regexes in searches.items():
                if regexes:
                    find_line = f"{r} "

                    if type == "d":
                        find_line += "-type d "
                        bash_find_var = f"FIND_DIR_{r[1:].replace('.','').replace('-','_').replace('{ROOT_FOLDER}','').upper()}"
                        self.bash_find_d_vars.add(bash_find_var)
                        all_folder_regexes += regexes
                    else:
                        bash_find_var = f"FIND_{r[1:].replace('.','').replace('-','_').replace('{ROOT_FOLDER}','').upper()}"
                        self.bash_find_f_vars.add(bash_find_var)
                        all_file_regexes += regexes

                    find_line += '-name \\"' + '\\" -o -name \\"'.join(regexes) + '\\"'
                    find_line = FIND_TEMPLATE.replace(FIND_LINE_MARKUP, find_line)
                    find_line = f"{bash_find_var}={find_line}"
                    finds.append(find_line)

        # Buid folder and files finds when searching in a custom folder
        all_folder_regexes = list(set(all_folder_regexes))
        find_line = '$SEARCH_IN_FOLDER -type d -name \\"' + '\\" -o -name \\"'.join(all_folder_regexes) + '\\"'
        find_line = FIND_TEMPLATE.replace(FIND_LINE_MARKUP, find_line)
        find_line = f"FIND_DIR_CUSTOM={find_line}"
        finds_custom.append(find_line)

        all_file_regexes = list(set(all_file_regexes))
        find_line = '$SEARCH_IN_FOLDER -name \\"' + '\\" -o -name \\"'.join(all_file_regexes) + '\\"'
        find_line = FIND_TEMPLATE.replace(FIND_LINE_MARKUP, find_line)
        find_line = f"FIND_CUSTOM={find_line}"
        finds_custom.append(find_line)

        return finds, finds_custom

    def __generate_storages(self) -> list:
        """Generate the storages to save the results per entry"""
        storages = []
        custom_storages = ["FIND_CUSTOM", "FIND_DIR_CUSTOM"]
        all_f_finds = "$" + "\\n$".join(list(self.bash_find_f_vars) + custom_storages)
        all_d_finds = "$" + "\\n$".join(list(self.bash_find_d_vars) + custom_storages)
        all_finds = "$" + "\\n$".join(list(self.bash_find_f_vars) + list(self.bash_find_d_vars) + custom_storages)

        for precord in self.ploaded.peasrecords:
            bash_storage_var = f"PSTORAGE_{precord.bash_name}"
            self.bash_storages.add(bash_storage_var)

            #Select the FIND_ variables to search on depending on the type files
            if all(frecord.type == "f" for frecord in precord.filerecords):
                storage_line = STORAGE_TEMPLATE.replace(STORAGE_LINE_MARKUP, all_f_finds)
            elif all(frecord.type == "d" for frecord in precord.filerecords):
                storage_line = STORAGE_TEMPLATE.replace(STORAGE_LINE_MARKUP, all_d_finds)
            else:
                storage_line = STORAGE_TEMPLATE.replace(STORAGE_LINE_MARKUP, all_finds)

            #Grep by filename regex (ended in '$')
            bsp = '\\.' #A 'f' expression cannot contain a backslash, so we generate here the bs need in the line below
            grep_names = f" | grep -E \"{'|'.join([frecord.regex.replace('.',bsp).replace('*', '.*')+'$' for frecord in precord.filerecords])}\""

            #Grep by searched folders
            grep_folders_searched = f" | grep -E \"^{'|^'.join(list(set([d for frecord in precord.filerecords for d in frecord.search_in])))}\"".replace("HOMESEARCH","GREPHOMESEARCH")

            #Grep extra paths. They are accumulative between files of the same PEASRecord
            grep_extra_paths = ""
            if any(True for frecord in precord.filerecords if frecord.check_extra_path):
                grep_extra_paths = f" | grep -E '{'|'.join(list(set([frecord.check_extra_path for frecord in precord.filerecords if frecord.check_extra_path])))}'"

            #Grep to remove paths. They are accumulative between files of the same PEASRecord
            grep_remove_path = ""
            if any(True for frecord in precord.filerecords if frecord.remove_path):
                grep_remove_path = f" | grep -v -E '{'|'.join(list(set([frecord.remove_path for frecord in precord.filerecords if frecord.remove_path])))}'"

            #Construct the final line like: STORAGE_MYSQL=$(echo "$FIND_DIR_ETC\n$FIND_DIR_USR\n$FIND_DIR_VAR\n$FIND_DIR_MNT" | grep -E '^/etc/.*mysql|/usr/var/lib/.*mysql|/var/lib/.*mysql' | grep -v "mysql/mysql")
            storage_line = storage_line.replace(STORAGE_LINE_EXTRA_MARKUP, f"{grep_remove_path}{grep_extra_paths}{grep_folders_searched}{grep_names}")
            storage_line = f"{bash_storage_var}={storage_line}"
            storages.append(storage_line)

        return storages

    def __generate_sections(self) -> dict:
        """Generate sections for records with auto_check to True"""
        sections = {}

        for precord in self.ploaded.peasrecords:
            if precord.auto_check:
                section = f'if [ "$PSTORAGE_{precord.bash_name}" ] || [ "$DEBUG" ]; then\n'
                section += f'  print_2title "Analyzing {precord.name.replace("_"," ")} Files (limit 70)"\n'

                for exec_line in precord.exec:
                    if exec_line:
                        section += "    " + exec_line + "\n"

                for frecord in precord.filerecords:
                    section += "    " + self.__construct_file_line(precord, frecord) + "\n"

                section += "fi\n"

                sections[precord.name] = section

        return sections

    def __construct_file_line(self, precord: PEASRecord, frecord: FileRecord, init: bool = True) -> str:
        real_regex = frecord.regex[1:] if frecord.regex.startswith("*") and len(frecord.regex) > 1 else frecord.regex
        real_regex = real_regex.replace(".","\\.").replace("*",".*")
        real_regex += "$"

        analise_line = ""
        if init:
            analise_line = 'if ! [ "`echo \\\"$PSTORAGE_'+precord.bash_name+'\\\" | grep -E \\\"'+real_regex+'\\\"`" ]; then if [ "$DEBUG" ]; then echo_not_found "'+frecord.regex+'"; fi; fi; '
            analise_line += 'printf "%s" "$PSTORAGE_'+precord.bash_name+'" | grep -E "'+real_regex+'" | while read f; do ls -ld "$f" 2>/dev/null | sed -${E} "s,'+real_regex+',${SED_RED},"; '

        #If just list, just list the file/directory
        if frecord.just_list_file:
            if frecord.type == "d":
                analise_line += 'ls -lRA "$f";'
            analise_line += 'done; echo "";'
            return analise_line

        if frecord.type == "f":
            grep_empty_lines = ' | grep -IEv "^$"'
            grep_line_grep = f' | grep -E {frecord.line_grep}' if frecord.line_grep else ""
            grep_only_bad_lines = f' | grep -E "{frecord.bad_regex}"' if frecord.bad_regex else ""
            grep_remove_regex = f' | grep -Ev "{frecord.remove_regex}"' if frecord.remove_regex else ""
            sed_bad_regex = ' | sed -${E} "s,'+frecord.bad_regex+',${SED_RED},g"' if frecord.bad_regex else ""
            sed_very_bad_regex = ' | sed -${E} "s,'+frecord.very_bad_regex+',${SED_RED_YELLOW},g"' if frecord.very_bad_regex else ""
            sed_good_regex = ' | sed -${E} "s,'+frecord.good_regex+',${SED_GOOD},g"' if frecord.good_regex else ""

            if init:
                analise_line += 'cat "$f" 2>/dev/null'
            else:
                analise_line += 'cat "$ff" 2>/dev/null'

            if grep_empty_lines:
                analise_line += grep_empty_lines

            if grep_line_grep:
                analise_line += grep_line_grep

            if frecord.only_bad_lines and not grep_line_grep:
                analise_line += grep_only_bad_lines

            if grep_remove_regex:
                analise_line += grep_remove_regex

            if sed_bad_regex:
                analise_line += sed_bad_regex

            if sed_very_bad_regex:
                analise_line += sed_very_bad_regex

            if sed_good_regex:
                analise_line += sed_good_regex

            analise_line += '; done; echo "";'
            return analise_line

        #In case file is type "d"
        if frecord.files:
            for ffrecord in frecord.files:
                ff_real_regex = ffrecord.regex[1:] if ffrecord.regex.startswith("*") and ffrecord.regex != "*" else ffrecord.regex
                ff_real_regex = ff_real_regex.replace("*",".*")
                #analise_line += 'for ff in $(find "$f" -name "'+ffrecord.regex+'"); do ls -ld "$ff" | sed -${E} "s,'+ff_real_regex+',${SED_RED},"; ' + self.__construct_file_line(precord, ffrecord, init=False)
                analise_line += 'find "$f" -name "'+ffrecord.regex+'" | while read ff; do ls -ld "$ff" | sed -${E} "s,'+ff_real_regex+',${SED_RED},"; ' + self.__construct_file_line(precord, ffrecord, init=False)

        analise_line += 'done; echo "";'
        return analise_line

    def __get_linux_exploit_suggesters(self) -> tuple:
        r1 = requests.get("https://raw.githubusercontent.com/mzet-/linux-exploit-suggester/master/linux-exploit-suggester.sh")
        r2 = requests.get("https://raw.githubusercontent.com/jondonas/linux-exploit-suggester-2/master/linux-exploit-suggester-2.pl")
        return(base64.b64encode(bytes(r1.text, 'utf-8')).decode("utf-8"), base64.b64encode(bytes(r2.text, 'utf-8')).decode("utf-8"))

    def __get_bin(self, url, tar_gz="") -> str:
        os.system(f"wget -q '{url}' -O /tmp/bin_builder")
        if tar_gz:
            os.system(f"cd /tmp; tar -xvzf /tmp/bin_builder; rm /tmp/bin_builder; mv {tar_gz} /tmp/bin_builder")
        os.system("base64 /tmp/bin_builder | tr -d '\n' > /tmp/binb64; rm /tmp/bin_builder")

        b64bin = ""
        with open("/tmp/binb64", "r") as f:
            b64bin = f.read()

        os.system("rm /tmp/binb64")
        return b64bin

    def __get_gtfobins_lists(self) -> tuple:
        r = requests.get("https://github.com/GTFOBins/GTFOBins.github.io/tree/master/_gtfobins")
        bins = re.findall(r'_gtfobins/([\w_ \-]+).md', r.text)

        sudoVB = []
        suidVB = []
        capsVB = []

        for b in bins:
            rb = requests.get(f"https://raw.githubusercontent.com/GTFOBins/GTFOBins.github.io/master/_gtfobins/{b}.md")
            if "sudo:" in rb.text:
                sudoVB.append(b+"$")
            if "suid:" in rb.text:
                suidVB.append("/"+b+"$")
            if "capabilities:" in rb.text:
                capsVB.append(b)

        return (suidVB, sudoVB, capsVB)

    def __generate_regexes_search(self) -> str:
        regexes = REGEXES_LOADED["regular_expresions"]

        regexes_search_section = ""

        for values in regexes:
            section_name = values["name"]
            regexes_search_section += f'    print_2title "Searching {section_name}"\n'

            for entry in values["regexes"]:
                name = entry["name"]
                caseinsensitive = entry.get("caseinsensitive", False)
                regex = entry["regex"]
                regex = regex.replace('"', '\\"').strip()
                falsePositives = entry.get("falsePositives", False)

                if falsePositives:
                    continue

                regexes_search_section += f"    search_for_regex \"{name}\" \"{regex}\" {'1' if caseinsensitive else ''}\n"

            regexes_search_section += "    echo ''\n\n"

        return regexes_search_section

    def __replace_mark(self, mark: str, find_calls: list, join_char: str):
        """Substitude the markup with the actual code"""

        self.linpeas_sh = self.linpeas_sh.replace(mark, join_char.join(find_calls)) #New line char is't needed

    def write_linpeas(self, path, rm_startswith=""):
        """Write on disk the final linpeas"""

        with open(path, "w") as f:
            if not rm_startswith:
                f.write(self.linpeas_sh)
            else:
                tmp_linpeas = ""
                for line in self.linpeas_sh.splitlines():
                    if not line.startswith(rm_startswith):
                        tmp_linpeas += line + "\n"
                f.write(tmp_linpeas)


from datetime import datetime
from django.shortcuts import render, redirect
from django.utils import timezone
from .models import Schedule, Route, Stop

# Create your views here.
def homeView(request):
    routes = Route.objects.all()
    context = {
        "routes": routes,
    }
    return render(request, "home/index.html", context)

def AllRoutes(request):
    AllRoutes = Route.objects.all()
    context = {
        "AllRoutes": AllRoutes,
    }
    return render(request, "home/all_routes.html", context)

def allSchedules(request, pk):
    route = Route.objects.get(id=pk)
    AllSchedules = Schedule.objects.filter(route=route).order_by("departure_time")

    context = {
        "AllSchedules": AllSchedules,
        "route": route,
    }
    return render(request, "home/all_schedules.html", context)

def allStops(request, pk):
    route = Route.objects.get(id=pk)
    AllStops = Stop.objects.filter(route=route).order_by("departure_time")
    context = {
        "AllStops": AllStops,
        "route": route,
    }
    return render(request, "home/all_stops.html", context)


# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# pylint: disable=unused-import, redefined-builtin

import shutil
import os
import sys
import re
import string
import datetime
from typer.testing import CliRunner

# Include this to avoid circular import errors
import prompt_toolkit
import sphinx_material

os.environ['MLTK_BUILD_DOCS'] = '1'

import mltk
from mltk import cli
from mltk import MLTK_DIR, MLTK_ROOT_DIR
from mltk.utils.path import (
    clean_directory,
    get_user_setting,
    recursive_listdir,
    fullpath,
    walk_with_depth
)

curdir = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.normpath(f'{curdir}/../../../mltk'))
repo_project_name = os.environ.get('MLTK_REPO_PROJECT_NAME', 'siliconlabs')
repo_url = f'https://github.com/{repo_project_name}/mltk'
repo_project_name = get_user_setting('repo_project_name', 'siliconlabs')

# -- Project information -----------------------------------------------------

project = 'MLTK'
copyright = f'{datetime.date.today().year}, Silicon Labs'
author = 'Silicon Labs'
release = mltk.__version__

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    "sphinx.ext.autodoc",
    'sphinx.ext.autosummary',
    'sphinx.ext.coverage',
    "numpydoc",
    "sphinx.ext.doctest",
    "sphinx.ext.extlinks",
    "sphinx.ext.intersphinx",
    "sphinx.ext.todo",
    "sphinx.ext.mathjax",
    "sphinx.ext.viewcode",
    'sphinx.ext.napoleon',
    #'sphinx.ext.autosectionlabel',
    "myst_nb",
     # "myst_parser", # this is not needed when myst_nb is used
    "sphinx_markdown_tables",
    "sphinx_copybutton",
    'sphinx_design',
    'sphinx_material',
    'sphinx_autodoc_typehints', # This must come last
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_static/templates']

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = [
    '_static',
    '*lib/site-packages',
    '*lib\\site-packages'
]

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_material'
html_logo = '_static/logo.png'
html_favicon = "_static/favicon.ico"

html_show_sourcelink = True
html_use_index = True
html_domain_indices = True
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

html_js_files = [
    'js/custom.js',
    'js/apitoc.js'
]
html_css_files = [
    'css/custom.css',
]

# https://bashtage.github.io/sphinx-material/customization.html#customization
html_theme_options = {
    # Set the name of the project to appear in the navigation.
    'nav_title': 'Machine Learning Toolkit',

    # Set you GA account ID to enable tracking
    'google_analytics_account': 'G-HZ5MW943WF',

    # Set the color and the accent color
    'color_primary': 'red',
    'color_accent': 'light-blue',

    'base_url': False,
    'repo_url': repo_url,
    'repo_name': 'MLTK Github Repository',

    # Visible levels of the global TOC;
    'globaltoc_depth': 2,
    # If False, expand all TOC entries
    'globaltoc_collapse': True,
    # If True, show hidden TOC entries
    'globaltoc_includehidden': True,
    'heroes': {
        'index': 'A Python package to aid the development of machine learning models for Silicon Lab\'s embedded platforms',
        'docs/installation': 'Install the MLTK Python Package via PIP or in <a href="https://colab.research.google.com/notebooks/welcome.ipynb" target="_blank">Google Colab<a/>',
        'docs/command_line': 'Execute the MLTK modeling features from the command-line',
        'docs/guides/index': 'The MLTK provides numerous features to aid the development of ML models',
        'docs/guides/model_profiler': 'Profile a model to determine how efficiently it may execute on hardware',
        'docs/guides/model_visualizer' : 'Visualize a model using an interactive webpage',
        'docs/guides/model_specification': 'Define a model specification using a standard Python script',
        'docs/guides/model_training': 'Train an ML model using Google Tensorflow',
        'docs/guides/model_evaluation': 'Evaluate an ML model to determine how accurate it is',
        'docs/guides/model_quantization': 'Quantize a model to reduce its memory footprint',
        'docs/guides/model_parameters': 'Embed custom parameters into the generated model file',
        'docs/guides/model_summary': 'Generate a textual summary of a model',
        'docs/guides/model_archive': 'All model development files are store in a distributable archive',
        'docs/guides/model_search_path': 'Specify the model search path',
    },
    'nav_links': [
        {
            'href': 'https://docs.silabs.com/gecko-platform/latest/machine-learning/tensorflow/overview',
            'internal': False,
            'title': 'Gecko SDK Documentation',
        },
        {
            'href': 'https://github.com/tensorflow/tflite-micro',
            'internal': False,
            'title': 'Tensorflow-Lite Micro Repository',
        },
        {
            'href': 'https://www.tensorflow.org/learn',
            'internal': False,
            'title': 'Tensorflow Documentation',
        },
    ],

    'html_minify': False,
    'html_prettify': False,
    'css_minify': False,

    'version_dropdown': False,
    'version_json': None,
    'version_info': None
}

html_sidebars = {
    '**': ['logo-text.html', 'globaltoc.html', 'localtoc.html', 'searchbox.html']
}

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
source_suffix = ['.rst', '.md']

pygments_style = 'trac'
add_module_names = False
autosummary_generate = True
#autosummary_imported_members = True
numpydoc_class_members_toctree = False

numpydoc_show_class_members = False
typehints_fully_qualified = False
set_type_checking_flag  = True

autodoc_typehints = 'description'
autodoc_member_order = 'bysource'
autoclass_content = "class"
autosectionlabel_prefix_document = True
autodoc_class_signature = 'separated'
#autosectionlabel_maxdepth = 1
myst_heading_anchors = 3
language = "en"
html_last_updated_fmt = ""

todo_include_todos = True

jupyter_execute_notebooks = 'off'

build_dir = sys.argv[-1]

# Copy all the .md files in <mltk root>/docs
# to <mltk root>/docs/website_builder/source/docs
# This allows for sphinx to generate HTML for each .md
docs_src_dir = fullpath(f'{curdir}/../..')
docs_dst_dir = f'{curdir}/docs'
clean_directory(docs_dst_dir)
include_re = re.compile(r'```{include}\s+(.*)\s*```.*')
cpp_path_re = re.compile(r'.*\]\((\.\.\/[\.\.\/]*cpp)[\/\)]+.*')
mltk_path_re = re.compile(r'.*\]\((\.\.\/[\.\.\/]*mltk\/core)[\/\)]+.*')
for fn in recursive_listdir(docs_src_dir, return_relative_paths=True):
    if not fn.endswith(('.md', '.rst')):
        continue
    if 'website_builder' in fn:
        continue

    src_path = f'{docs_src_dir}/{fn}'
    dst_path = f'{docs_dst_dir}/{fn}'
    os.makedirs(os.path.dirname(dst_path), exist_ok=True)

    # First check if the file contains:
    # ```{include} <path>
    #
    # If it does, then update the source path to that path.
    # This way we can process the actual markdown file
    with open(src_path, 'r') as f:
        data = f.read().replace('\r', '').replace('\n', '')
        match = include_re.match(data)
        if match:
            relpath = match.group(1)
            src_path = f'{os.path.dirname(src_path)}/{relpath}'

    # Now process the markdown file
    with open(src_path, 'r') as f:
        data = ''
        for line in f:
            if 'Refer to the [online documentation](https://siliconlabs.github.io/mltk) to properly view this file' in line:
                continue

            # Update the relative links to C++ code to point to the github repo
            # https://github.com/siliconlabs/mltk/cpp
            match = cpp_path_re.match(line)
            if match:
                line = line.replace(match.group(1), 'https://github.com/siliconlabs/mltk/tree/master/cpp/')
            else:
                # Update the relative links to python code to point to the github repo
                # https://github.com/siliconlabs/mltk/mltk/core
                match = mltk_path_re.match(line)
                if match:
                    line = line.replace(match.group(1), 'https://github.com/siliconlabs/mltk/tree/master/mltk/core/')
            data += line

    with open(dst_path, 'w') as f:
        f.write(data)

# Generate a markdown file containing each command's CLI --help
runner = CliRunner()
cli.create_cli()
for cmd in (
    'profile', 'train', 'tensorboard', 'ssh',
    'evaluate', 'quantize', 'summarize', 'view',
    'update_params', 'view_audio', 'classify_audio',
    'classify_image', 'fingerprint_reader', 'commander'
):
    result = runner.invoke(cli.root_cli, [cmd, '--help'], color=False, prog_name='mltk')
    help_str = ''.join(c for c in result.stdout if c in string.printable)
    with open(f'{docs_dst_dir}/command_line/{cmd}_cli_help.md', 'w') as f:
        f.write('```text\n')
        f.write(help_str)
        f.write('```\n')

# Copy all the files in <mltk root>/docs/img
# to <mltk root>/docs/website_builder/source/docs/img
# This allows for relative referencing
img_src_dir = f'{curdir}/../../img'
img_dst_dir = f'{curdir}/docs/img'
os.makedirs(img_dst_dir, exist_ok=True)
clean_directory(img_dst_dir)
shutil.copytree(img_src_dir, img_dst_dir, dirs_exist_ok=True)

# Copy <mltk root>/docs/img/models -> <mltk root>/docs/_images/models
# We need to do this because the images are referenced as raw HTML in
# each model python script header comments which won't be processed by sphinx
img_src_dir = f'{curdir}/../../img/models'
img_dst_dir = f'{curdir}/../../_images/models'
os.makedirs(img_dst_dir, exist_ok=True)
clean_directory(img_dst_dir)
shutil.copytree(img_src_dir, img_dst_dir, dirs_exist_ok=True)

# Copy each .ipynb or .md file in <mltk root>/mltk/examples
# to <mltk root>/docs/website_builder/source/mltk/examples directory
# Also update the Github project name as necessary
EXAMPLE_SRC_DIS = [
    'mltk/examples',
    'mltk/tutorials',
    'cpp/shared/uart_stream'
]

url_re = re.compile(r'.*\((https:\/\/siliconlabs\.github\.io\/mltk\/).*', re.I)
url_re2 = re.compile(r'.*\((https:\/\/siliconlabs\.github\.io\/mltk\/).*(\.html).*', re.I)
docs_img_re = re.compile(r'.*\((https:\/\/github.com\/SiliconLabs\/mltk\/raw\/master\/docs\/img\/).*', re.I)
# If the line contains something like:
# (https://siliconlabs.github.io/mltk/docs/python_api/mltk_model.html#mltk.core.TrainMixin.tflite_converter)
# Then we cannot convert it to it's corresponding relative markdown URL, so it's not supported
not_supported_url_re = re.compile(r'.*\(https:\/\/siliconlabs\.github\.io\/mltk\/.*\.html#.*[\.\-].*\)', re.I)

for src_dir in EXAMPLE_SRC_DIS:
    for root, _, files in walk_with_depth(f'{MLTK_ROOT_DIR}/{src_dir}', depth=10):
        for fn in files:
            if not fn.endswith(('.ipynb', '.md')):
                continue
            rel_dir = os.path.relpath(root, MLTK_ROOT_DIR)
            dst_dir = f'{curdir}/{rel_dir}'
            os.makedirs(dst_dir, exist_ok=True)
            shutil.copy(f'{root}/{fn}', f'{dst_dir}/{fn}')
            data = ''
            with open(f'{dst_dir}/{fn}', 'r', encoding='utf-8') as fp:
                for line in fp:
                    # The notebooks use full URLs.
                    # Convert the URLs to a relative path when we're generating
                    # so we can generate HTML with relative links
                    match = url_re.match(line)
                    not_supported_match = not_supported_url_re.match(line)
                    rel_path_to_root = os.path.relpath(curdir, dst_dir).replace('\\', '/') + '/'
                    if not_supported_match is None and match:
                        match2 = url_re2.match(line)
                        line = line.replace(match.group(1), rel_path_to_root)
                        if match2:
                            line = line.replace(match2.group(2), '.md')

                    match = docs_img_re.match(line)
                    if match:
                        line = line.replace(match.group(1), f'{rel_path_to_root}docs/img/')

                    data += line + '\n'

            with open(f'{dst_dir}/{fn}', 'w', encoding='utf-8') as fp:
                fp.write(data)

# Copy the <mltk root>/mltk/core/tflite_model_parameters/schema/dictionary.fbs
# to the docs build directory
dictionary_fbs_src = f'{MLTK_DIR}/core/tflite_model_parameters/schema/dictionary.fbs'
dictionary_fbs_dst = f'{docs_dst_dir}/python_api/tflite_model/dictionary.fbs'
dictionary_fbs_build_dir = f'{build_dir}/docs/python_api/tflite_model/dictionary.fbs'
shutil.copy(dictionary_fbs_src, dictionary_fbs_dst)
os.makedirs(os.path.dirname(dictionary_fbs_build_dir), exist_ok=True)
shutil.copy(dictionary_fbs_src, dictionary_fbs_build_dir)

def autodoc_skip_member(app, what, name, obj, skip, opts):
    # we can document otherwise excluded entities here by returning False
    # or skip otherwise included entities by returning True
    qual_name = getattr(obj, '__qualname__', '')
    module = getattr(obj, '__module__', '')
    fullname = f'{module}.{qual_name}'

    if 'keras.preprocessing.image.ImageDataGenerator.flow_from_dataframe' in fullname:
        return True

    return None

def setup(app):
    app.connect('autodoc-skip-member', autodoc_skip_member)


from connection import create_connection
from main import search_recommended
import csv

database = create_connection()

cursor = database[0]
connection = database[1]
file = 'csv/profile_recommendations.csv'

# this program is a little more hardcoded since it used more tables instead of one

def search_profiles(r_list):
    # get the profile id's from the session instead of all the data
    query = """SELECT profid FROM sessions WHERE id = %s"""
    p_list = []

    for item in r_list:
        cursor.execute(query, (item, ))
        row = cursor.fetchone()
        p_list.append(row)

    return p_list

def search_products(profiles):
    query = """SELECT prodid FROM profiles_previously_viewed WHERE profid = %s LIMIT 2"""

    # get the first profile..this is the profile u want products for
    recommended_list = []
    prof_id = ''.join(profiles[0])
    recommended_list.append(prof_id)

    # for all the other profiles..search product viewed with query
    for i in range(1, len(profiles)):
        cursor.execute(query, (profiles[i], ))
        row = cursor.fetchone()
        # some profiles apparently never visited a product so in this case append none
        if row is None and len(recommended_list) < 4:
            recommended_list.append(row)
        elif row not in recommended_list and len(recommended_list) < 4:
            row = ''.join(row)
            recommended_list.append(row)
        else:
            continue

    return recommended_list

def session_comparison():
    query = """SELECT * FROM sessions LIMIT 5000"""
    cursor.execute(query)
    rows = cursor.fetchall()
    count = 0

    with open(file, 'w', newline='') as write_file:
        writer = csv.writer(write_file, delimiter=',')
        writer.writerow(['profid', 'prod_id_1', 'prod_id_2', 'prod_id_3'])
        # for each profile use functions
        for row in rows:
            # first get sessions that look alike based on:
            r_list = search_recommended('sessions', ['os', 'devicetype'], row)
            # get the profiles that are connected to these sessions
            profiles = search_profiles(r_list)
            # get the products these profile once visited
            products = search_products(profiles)
            # write this to file
            writer.writerow(products)
            # count to check progress
            count += 1
            print(count)

session_comparison()

cursor.close()
connection.close()


import pandas as pd
from matplotlib.colors import ListedColormap
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.metrics import accuracy_score
#https://scikit-learn.org/stable/modules/model_evaluation.html
import matplotlib.pyplot as plt

import numpy as np
from numpy import mean

from sklearn.svm import SVC
from sklearn.svm import LinearSVC

#Auxiliar function that reformats df to one that is more fitting to Classification Problems
def reformat(dataFrame):
    dataFrame['y']=dataFrame.columns[0]
    dataFrame.rename(columns={dataFrame.columns[0]: 'word'}, inplace=True)
    return dataFrame;
#RawData, it has been a little bit formatted before reading the csv, to facilitate the process.
raw=pd.read_csv("data/data.csv")

#Separate df
catala, angles= raw.filter(['catala'], axis=1), raw.filter(['angles'], axis=1)
#Reformating
catala=reformat(catala)
angles=reformat(angles)
#Merging
wordsDF=pd.concat([catala,angles], axis=0)
#Shuffle the rows
wordsDF = wordsDF.sample(frac=1).reset_index(drop=True)

#ratio de consonantes y vocales
def ratio (word):
    vocals=0
    for c in word:
        if isVocal(c):
            vocals+=1
    return vocals/len(word)

#isVocal?
def isVocal(c):
    if(c=='a' or c=='e' or c=='i' or c=='o' or c=='u'):
        return True
    return
#gotAccent?
def gotAccent(word):
    #List containing all possible accentuated chars from Catalan
    accentuatedChars=[ord('à'),ord('è'),ord('é'),ord('í'),ord('ò'),ord('ó'),ord('ú')]
    for c in word:
        if ord(c) in accentuatedChars:
            return 1
    return 0
def doubleVocal(word):
    ocurrences=["aa","ee","ii","oo","uu"]
    for oc in ocurrences:
        if word.find(oc)!=-1:
            return 1
    return 0

def enCC(word):
    ocurrences=["sch","spl","shr","squ","thr","spr","scr","sph","th","tw","sw","sk","sm"]
    for oc in ocurrences:
        if word.find(oc)!=-1:
            return 1
    return 0
'''
def doubleVocal(word):
    possibleDuo=False
    vocal = ' '
    for c in word:
        if possibleDuo==True:
            #Possible Duo Found?
            if isVocal(c):
                if c == vocal:
                    #Duo Found
                    return 1
                else:
                    #New Possible Duo
                    vocal = c
            #Not Found
            else:
                possibleDuo=False
        else:
            if isVocal(c):
                #Possible Duo in the future
                vocal=c
                possibleDuo=True
    return 0
'''

wordsDF['ratio']=wordsDF['word'].apply(ratio)
wordsDF['cantidadLetras']=wordsDF['word'].apply(len)
wordsDF['gotAccent']=wordsDF['word'].apply(gotAccent)
wordsDF['doubleVocal']=wordsDF['word'].apply(doubleVocal)
wordsDF['enCC']=wordsDF['word'].apply(enCC)
wordsDF=wordsDF[['word','ratio','cantidadLetras','gotAccent','doubleVocal','enCC','y']]
wordsDF.to_csv('data/definitiveData.csv', index=False)
print(wordsDF)

#First we must separate dataframe into X and y format
X=wordsDF.iloc[:,1:6]
y=wordsDF.iloc[:,6]
print(X)
#Then we separate the data frame in training and test (will be used in chosen model)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=33)
#Modelo
SVM_Lineal = LinearSVC(loss='squared_hinge', dual=False, C=50)

def evaluate_model(cv):
    #https://scikit-learn.org/stable/modules/cross_validation.html#cross-validation
    score = cross_val_score(SVM_Lineal,X,y,scoring='accuracy',cv=cv,n_jobs=-1)
    return mean(score),score.min(),score.max()

#Range to be tested
folds = range (5,11)
for k in folds:
    cv=StratifiedKFold(n_splits=k, shuffle=True)
    k_mean, k_min, k_max = evaluate_model(cv)
    print(f'-> folds={k}, accuracy = {round(k_mean,4)}, ({round(k_min,4)}, {round(k_max,4)})')


# https://www.practicepython.org/exercise/2014/03/26/08-rock-paper-scissors.html

while True:
    # player 1 choice
    while True:
        player1 = input('Player1: Rock, paper, or scissors?\n')
        if player1 != 'rock' and player1 != 'paper' and player1 != 'scissors':
            continue
        print('Player1 chose', player1)
        break

    # player 2 choice
    while True:
        player2 = input('Player2: Rock, paper, or scissors?\n')
        if player2 != 'rock' and player2 != 'paper' and player2 != 'scissors':
            continue
        print('Player2 chose', player2)
        break

    if player1==player2:
        print('Draw!')
    elif player1=='rock' and player2=='scissors':
        print('Player 1 wins')
    elif player1=='scissors' and player2=='paper':
        print('Player 1 wins')
    elif player1=='paper' and player2=='rock':
        print('Player 1 wins')
    elif player1=='scissors' and player2=='rock':
        print('Player 2 wins')
    elif player1=='paper' and player2=='scissors':
        print('Player 2 wins')
    elif player1=='rock' and player2=='paper':
        print('Player 2 wins')

    print('Play again? (y/n)')

    replay = input()

    if replay != 'y' and replay != 'n':
        continue
    if replay == 'n':
        print('Goodbye.')
        break


def quicksort(arr, left, right):
    '''
    Implementation of the quicksort algoritm
    '''

    if left >= right:
        return

    pivot = arr[(left + right)/2]

    index = partition(arr, left, right, pivot)

    quicksort(arr, left, index - 1)
    quicksort(arr, index, right)

def partition(arr, left, right, pivot):

    while(left <= right):
        while(arr[left] < pivot):
            left += 1

        while(arr[right] > pivot):
            right -= 1

        if (left <= right):
            #swap(arr, left, right)
            arr[left], arr[right] = arr[right], arr[left]

            left += 1
            right -= 1

    return left


from urllib.request import urlopen
from html.parser import HTMLParser

def isjob(url):
    try:
        a, b, c, d = url.split('/')
    except ValueError:
        return False
    return a == d == '' and b == 'jobs' and c.isdigit()

class Scraper(HTMLParser):

    in_link = False

    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        url = attrs.get('href', '')
        if tag == 'a' and isjob(url):
            self.url = url
            self.in_link = True
            self.chunks = []

    def handle_data(self, data):
        if self.in_link:
            self.chunks.append(data)

    def handle_endtag(self, tag):
        if tag == 'a' and self.in_link:
            print('{} ({})'.format(''.join(self.chunks), self.url))
            self.in_link = False

text = urlopen('http://python.org/jobs').read().decode()
parser = Scraper()
parser.feed(text)
parser.close()


"""
汉诺塔
补充：
汉诺塔的游戏思路其实就是怎么把大象关进冰箱：
1：打开冰箱的门（移动n-1个盘子到临时位）
2：把大象塞进去（把第n个盘子塞到目标位）
3：把门关上（移动n-1个盘子从临时位到目标位)
"""
count = 0
def hanoi(n,src,dst,mid):
    """汉诺塔

    Arguments:
        n {int} -- num of plates
        src {str} -- start node
        dst {str} -- target node
        mid {str} -- temp node
    """
    global count
    if n == 1:
        print("{}:{}->{}".format(1,src,dst))
    else:
        hanoi(n-1,src,mid,dst)
        print("{}:{}->{}".format(n,src,dst))
        count+=1
        hanoi(n-1,mid,dst,src)

hanoi(3,"A","C","B")


from pathlib import Path

def find_data_dir():
    cur_dir = Path('.').resolve()
    data_dir = _find_data_in_dir(cur_dir)
    return data_dir

def get_data_path(name):
    data_dir = find_data_dir()
    data_path = data_dir / name
    if not data_path.exists():
        raise IOError('No data file named {}'.format(name))
    return str(data_path)

def _find_data_in_dir(path):
    for child in path.iterdir():
        if child.is_dir() and child.name == 'data' and _has_root_files(child):
            return child
    parent = path.parent
    if parent != Path.home():
        return _find_data_in_dir(parent)

def _has_root_files(path):
    for child in path.iterdir():
        if child.is_file() and child.name.endswith('.root'):
            return True
    return False


from dash import Dash, html ,dcc
from . import dropdown, bar_chart,navbar, all_line,all_stop,direction, real_data_chart, map, scores_table, statistics, stop_score_chart,map,reset, date_button_dropdown
import plotly.express as px
from dash.dependencies import Output, Input,MATCH,State
from . import ids

import dash

def create_layout(app: Dash) -> html.Div:
    return html.Div(
        className= "app-div",
        children=[
            dcc.Store(id=ids.SELECTED_LINE),
            navbar.render(app),
            all_line.render(app),
            direction.render(app),
            create_dropdown_menu(app),
            html.H1("Headways",id="headway"),
            bar_chart.render(app),
            real_data_chart.render(app),
            html.H1("Statistics",id="statistics"),
            statistics.render(app),
            date_button_dropdown.render(app),
            scores_table.render(app),
            # html.Div(id= ids.DIV_MAP),
            html.H1("Map",id="about"),
            map.display_all_map()
        ]
    )

def create_dropdown_menu(app):

    return  html.Div(className="dropdown-menu",children=[
                dropdown.stop_id_render(app),
                dropdown.day_render(app),
                dropdown.date_render(app),
                dropdown.real_date_render(app),
                reset.render(app)
                ],
     )

    #  style = {"align-items": "center", "justify-content": "center"}


import errno
import io
import logging
import os
import pwd
import shutil
import subprocess
import tempfile
import textwrap
import time

import paramiko

from sparkdl.utils import logwrap, _get_max_num_concurrent_tasks

class IOUtils(object):
    """
    Utility class to manage directories and files.
    """
    @staticmethod
    def makedirs_exist_ok(name):
        """
        Recursively creates directories, and skips if directory already exists.
        This is similar to os.makedirs(name, exist_ok=True) in Python 3.
        :param name: dir name
        """
        try:
            os.makedirs(name)
        except OSError:
            # Tasks on the same worker would attempt to create the same dir.
            # In case of error, we just need to confirm the dir already exists.
            if not os.path.isdir(name):
                raise

class SshSessionManager(object):
    """
    Helper class to manage an SSH session per run.
    It assumes the SSH setup is for the current user.
    """
    def __init__(self, run_dir, _home_dir=None, _configure_ssh=True):
        """
        :param run_dir: working dir for the run, where we write the private key
        :param _home_dir: home directory (for test only)
        :param _configure_ssh: whether to configure SSH (for test only)
        """
        self._run_dir = run_dir
        self._home_dir = _home_dir or SshSessionManager._get_home_dir()
        self._configure_ssh = _configure_ssh
        self._generate_keypair()
        self.private_key_path = os.path.join(self._run_dir, "id_rsa")

    @staticmethod
    def _get_home_dir():
        """
        Returns the home dir of the current user.
        """
        return pwd.getpwuid(os.getuid()).pw_dir

    def _generate_keypair(self):
        """
        Generates and stores an SSH key pair.
        """
        pair = paramiko.RSAKey.generate(2048)
        private_key_out = io.StringIO()
        pair.write_private_key(private_key_out)
        self._private_key = private_key_out.getvalue()
        self._public_key = "ssh-rsa %s" % pair.get_base64()

    def write_private_key(self):
        """
        Writes the private key under the run dir and returns its path.
        This method is called by the driver and the first executor.
        """
        if not self._configure_ssh:
            return None
        assert os.path.isdir(self._run_dir)
        with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
            f.write(self._private_key)
        shutil.move(f.name, self.private_key_path)
        os.chmod(self.private_key_path, 0o400)
        return self.private_key_path

    def _get_public_key_entry(self):
        """Returns the public key entry to be added to authorized_keys."""
        return textwrap.dedent("""
        # Added by HorovodRunner
        {}
        """.format(self._public_key))

    def authorize_public_key(self):
        """
        Adds the public key to authorized_keys.
        This method is called by all executors.
        """
        if not self._configure_ssh:
            return
        ssh_dir = os.path.join(self._home_dir, ".ssh")
        IOUtils.makedirs_exist_ok(ssh_dir)
        authorized_keys_path = os.path.join(ssh_dir, "authorized_keys")
        entry = self._get_public_key_entry()
        with open(authorized_keys_path, 'a') as f:
            # We assume the underlying file system is POSIX-compliant.
            # The content is less than 512 bytes and hence less than PIPE_BUF.
            # So the append should be atomic without explict locks.
            # See write(2) and pipe(7).
            f.write(entry)

    def mask_authorized_public_key(self):
        """
        Masks the public key previously added to authorized_keys by "#"s of the same length.
        So the offsets of the key entries are not changed.
        This method is called by all executors after each run.
        """
        if not self._configure_ssh:
            return
        authorized_key_path = os.path.join(self._home_dir, ".ssh", "authorized_keys")
        target = self._public_key + "\n"
        mask = "#" * len(self._public_key) + "\n"
        with open(authorized_key_path, "r+") as f:
            # We do not use "for line in f:" because it has a read-ahead buffer.
            # See https://docs.python.org/2/library/stdtypes.html#file.next
            pos = f.tell()
            line = f.readline()
            while line:
                if line == target:
                    f.seek(pos)
                    f.write(mask)
                pos = f.tell()
                line = f.readline()

    def clean_private_key(self):
        """
        Clean up the private key under the run dir.
        This method is called by the driver and the first executor.
        """
        if not self._configure_ssh:
            return
        try:
            os.remove(self.private_key_path)
        except OSError as ex:
            # In CE and local mode, there are two delete attempts,
            # one from driver and one from worker #0.
            if ex.errno != errno.ENOENT:
                raise

    @staticmethod
    def setup_sshd():
        """
        Start the sshd if it is not running
        """
        # "service ssh status" will return 0, if ssh is running.
        if os.system("service ssh status"):
            logging.warning("SSH is not running. Start SSH service.")
            os.system("service ssh start")

def inherit_doc(cls):
    """
    A decorator that makes a class inherit documentation from its parents.
    """
    for name, func in vars(cls).items():
        # only inherit docstring for public functions
        if name != "__init__" and name.startswith("_"):
            continue
        if not func.__doc__:
            for parent in cls.__bases__:
                parent_func = getattr(parent, name, None)
                if parent_func and getattr(parent_func, "__doc__", None):
                    func.__doc__ = parent_func.__doc__
                    break
    cls.__doc__ = cls.__bases__[0].__doc__
    return cls

def check_and_get_num_partitions(sc, timeout_seconds=2 * 60):
    """
    Gets total number of partitions possible concurrently on a given spark context and raises
    an exception if there are no executors in timeout_seconds.
    """
    WARN_INTERVAL = 5
    CHECK_INTERVAL = 0.1
    current_time = time.time()
    start_time = current_time
    warn_time = current_time
    end_time = current_time + timeout_seconds
    while current_time < end_time:
        num_partitions = _get_max_num_concurrent_tasks(sc)
        if num_partitions > 0:
            return num_partitions
        if current_time >= warn_time:
            logging.warning(logwrap.fill("""
                Waiting for executor registration before HorovodRunner could start.
                {} seconds / {} seconds
                """.format(int(current_time-start_time), timeout_seconds)))
            warn_time += WARN_INTERVAL
        time.sleep(CHECK_INTERVAL)
        current_time = time.time()

    raise RuntimeError(logwrap.fill("""
        The cluster has no workers. At least 1 worker is needed to use
        HorovodRunner with np=0. Please increase your cluster size and retry.
        """))

def get_num_gpus():
    """
    Returns the number of gpus on a machine.
    """
    # This is necessary on GCP because nvidia-smi is in /usr/local/nvidia/bin,
    # which is not on the PATH. It has no effect on other clouds.
    set_path = "PATH=/usr/local/nvidia/bin:$PATH"
    result = subprocess.run(["bash", "-c", f"{set_path} nvidia-smi -L | wc -l"],
                            capture_output=True, check=False)
    # On CPU instances, nvidia-smi is unavailable. wc receives empty stdout and returns 0.
    num_gpus = int(result.stdout)
    return num_gpus

def get_gpu_amount_per_task(sc):
    return sc.getConf().get("spark.task.resource.gpu.amount")

def get_slots_per_partition(sc):
    """
    Returns number of slots we can have per partition, i.e.,
    return number of spark config value "spark.task.resource.gpu.amount" on a GPU cluster,
    or return 1 on a CPU cluster.
    If the cluster has no workers, it will raise a runtime error.
    """
    check_and_get_num_partitions(sc)
    gpu_per_task_str = get_gpu_amount_per_task(sc)
    if gpu_per_task_str is None:
        # not on a gpu cluster
        return 1
    else:
        return int(gpu_per_task_str)


from utils.quicksort import quickSort
from app.Node import Node
from app.Graph import Graph
from utils.fileReader import readGraph
import sys
import time
import math

def main(args):
  if(len(args) > 1):
    print('------------')
    print('Round: ', args[1])
    print('------------')
  numNodes = 0
  numEdges = 0
  graph = Graph()
  filename = 'school1'
  edges = readGraph("in/{filename}".format(filename=filename))
  for edge in edges:
    graph.addEdgeToGraph(edge[0], edge[1])

  startTimeGreedy = time.time()
  greedyColors = graph.greedyAlgorithm()
  print('Cores Guloso: ', greedyColors)
  endTimeGreedy = time.time()
  execTimeGreedy = round((endTimeGreedy - startTimeGreedy)*1000, 3)
  print("--- %f milliseconds ---" % execTimeGreedy)

  startTimeSequential = time.time()
  sequentialColors = graph.sequential()
  print('Cores Força bruta: ', sequentialColors)
  endTimeSequential = time.time()
  execTimeSequential = round((endTimeSequential - startTimeSequential)*1000, 3)
  print("--- %f milliseconds ---" % execTimeSequential)

  with open('out/{filename}-out.csv'.format(filename=filename), 'a') as outFile:
    outFile.write('{greedyColors};{execTimeGreedy};{sequentialColors};{execTimeSequential} \n'
      .format(
        greedyColors=greedyColors,
        execTimeGreedy=str(execTimeGreedy).replace('.',','),
        execTimeSequential=str(execTimeSequential).replace('.',','),
        sequentialColors=sequentialColors)
    )

  # graph.printGraph() # Imprime o grafo

main(sys.argv)


from matplotlib.pyplot import imread
import numpy as np
from glob import glob
from os.path import join

import torch as th
from torch.utils.data import Dataset

import sys

'''
Currently this module holds Sintel dataset processor only.
To use it for another dataset, specific classes must be accomplished.
'''

# here must be a path to Sintel script
# for "get_depth" function in the "sintel_io" module
path2SintelModule = './../DATASETS/Sintel/depth_training_20150305/sdk/python/'
sys.path.append(path2SintelModule)
import sintel_io

from utils.functional import get_training_sample, get_full_sample

class Sintel(Dataset):
    '''
    Allows to work with raw files, RGB and Depth maps.
    '''
    def __init__(self,
        path2images='./../DATASETS/Sintel/training_images/training/final/',
        path2depths='./../DATASETS/Sintel/depth_training_20150305/training/depth/',
        m=1, num_images=None, get_name=False, size=(400,400),\
        mode=None, do_transforms=True, crop_size=256, sample_mode='ordinary'):
        '''
        "mode" can be: "train" - returns three images,
                       None - returns just numpy-like RGB and Depth images
        '''
        self.size=size
        self.m = m
        self.path2images = path2images
        self.path2depths = path2depths

        self.depth_max = 1e2

        self.image_paths = sorted(glob(path2images+'*/*.png'))
        self.depth_paths = sorted(glob(path2depths+'*/*.dpt'))
        self.get_name = get_name

        if not ((mode == 'train') or (mode == 'val')):
            raise Exception(
                'Incorrect mode type! Only "train" or "val" are available.')
        self.mode = mode
        self.sample_mode = sample_mode
        self.do_transforms = do_transforms
        self.crop_size = crop_size

        if num_images is not None:
            self.image_paths = self.image_paths[:num_images]
            self.depth_paths = self.depth_paths[:num_images]

    def __len__(self):
        return len(self.image_paths)

    def transforms(self, image, depth):

        assert self.crop_size < min(depth.shape), \
        'Inconsistent crop size, it is bigger than image dimensions!'
        y0 = np.random.randint(0,image.shape[0] - self.crop_size)
        x0 = np.random.randint(0,image.shape[1] - self.crop_size)

        image = image[y0:y0+self.crop_size, x0:x0+self.crop_size]
        depth = depth[y0:y0+self.crop_size, x0:x0+self.crop_size]
        return image, depth

    def __getitem__(self, idx):
        image = imread(self.image_paths[idx]).astype(np.float32)
        depth = sintel_io.depth_read(self.depth_paths[idx])

        image /= image.max()
        depth[depth>self.depth_max] = self.depth_max

        file_name = self.image_paths[idx][len(self.path2images):-4]
        file_name = file_name.replace('/', '_')

        if self.mode == 'train':
            if self.do_transforms:
                image, depth = self.transforms(image, depth)
            if self.sample_mode == 'ordinary':
                out = get_training_sample(image, depth, m=self.m)
                out = [th.FloatTensor(elem)[None,...] for elem in out]
            elif self.sample_mode == 'full':
                out = get_full_sample(image, depth)
        elif self.mode == 'val':
            if self.size is not None:
                image = image[:self.size[0],:self.size[1]]
                depth = depth[:self.size[0],:self.size[1]]
            out = [image, depth]

        if self.get_name:
            return out+[file_name]
        else:
            return out

class SintelCropped_hf(Dataset):
    '''
    Allows to construct necessary low/high-spectrum images.
    MSGNet training requires high-freq parts only.
    '''
    def __init__(self, path2folder='./../DATASETS/Sintel/CUSTOM_TRAINING_m3/',\
        mode='train', num_images=None, get_name=False):

        self.m = int(path2folder[-2:][0])
        self.mode = mode
        self.path2folder = path2folder+mode+'/'
        self.path2D_hr_hf_est = self.path2folder+'D_hr_hf_est/'
        self.path2D_lr_hf     = self.path2folder+'D_lr_hf/'
        self.path2Y_hr_hf     = self.path2folder+'Y_hr_hf/'

        self.D_hr_hf_est_paths = sorted(glob(self.path2D_hr_hf_est+'*.pth'))
        self.D_lr_hf_paths     = sorted(glob(self.path2D_lr_hf    +'*.pth'))
        self.Y_hr_hf_paths     = sorted(glob(self.path2Y_hr_hf    +'*.pth'))

        self.get_name = get_name

        if num_images is not None:
            self.D_hr_hf_est_paths = self.D_hr_hf_est_paths[:num_images]
            self.D_lr_hf_paths     = self.D_lr_hf_paths    [:num_images]
            self.Y_hr_hf_paths     = self.Y_hr_hf_paths    [:num_images]

    def __len__(self):
        return len(self.D_hr_hf_est_paths)

    def __getitem__(self, idx):
        D_hr_hf_est = th.load(self.D_hr_hf_est_paths[idx])
        D_lr_hf     = th.load(self.D_lr_hf_paths[idx])
        Y_hr_hf     = th.load(self.Y_hr_hf_paths[idx])

        file_name = self.D_hr_hf_est_paths[idx][len(self.path2D_hr_hf_est):-4]

        out = [D_hr_hf_est, D_lr_hf, Y_hr_hf]

        if self.get_name:
            return out+[file_name]
        else:
            return out

class Sintel_FullTraining(Dataset):
    '''
    Rewritten SintelCropped_hf. Now it allows to get any pack of spectra and scales
    (based on "training_type" choice).
    '''
    def __init__(self, mode='train', num_images=None,
        get_name=False, training_type='ordinary'):

        # here must be a path to folder with prepared dataset. Should be a parameter in this function.
        path2dataset='./../DATASETS/Sintel/all_scales_freqs_training_set/'
        self.mode = mode
        self.path2mode_folder = path2dataset+mode+'/'
        self.paths2folders = sorted(glob(self.path2mode_folder+'*'))
        self.folders = [folder[len(self.path2mode_folder):]
                            for folder in self.paths2folders]
        self.paths2files = {folder:sorted(glob(path2folder+'/*.pth'))
                                for folder, path2folder in zip(self.folders, self.paths2folders)}

        self.get_name = get_name
        self.training_type = training_type
        #Possible values: "full_set"|"ordinary"|"multiloss"|"full_freq"

        if self.training_type == 'ordinary': # as in paper, hf-to-hf training
            self.get_from_folders = ['Y_hf', 'Dx1_hf', 'Dx8_hf']
        elif self.training_type == 'multiloss': # hf-to-hfs training (at each scale of upsampling)
            self.get_from_folders = ['Y_hf', 'Dx1_hf', 'Dx2_hf', 'Dx4_hf', 'Dx8_hf']
        elif self.training_type == 'full_freq': # hf-to-(hf+lf) training,
                                                # it was used to be able to clip network output by depth_max value.
            self.get_from_folders = ['Y_hf', 'Dx1_hf', 'Dx8_hf', 'Dx8_lf']
        elif self.training_type == 'full_set': # independent from training, just full pack of all possible images.
            self.get_from_folders = self.folders.copy()
        else:
            assert False, 'Incorrect type of training.'

        if num_images is not None:
            for folder in self.folders:
                self.paths2files[folder] = self.paths2files[folder][:num_images]

    def __len__(self):
        return len(self.paths2files['Y_hf'])

    def __getitem__(self, idx):
        out = {}

        for folder in self.get_from_folders:
            path2file = self.paths2files[folder][idx]
            out[folder] = th.load(path2file)

        file_name = path2file[len(self.path2mode_folder+'/'+folder):-4]
        if self.get_name:
            return [out, file_name]
        else:
            return out


import asyncio
import websockets
import threading
import json
import os

from pengelola_basisdata import PengelolaBasisdata

ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
CONFIG_PATH = os.path.join(ROOT_DIR, 'config-dev.json')

class PenyediaDataRealtime():
    def __init__(self):
        with open(CONFIG_PATH) as json_data_file:
            self.config = json.load(json_data_file)

        self.loop = None
        self.pengelola_basisdata = PengelolaBasisdata()

    def start(self):
        print("starting websocket")
        self.start_server = websockets.serve(
            self.server_websocket, "localhost", 6789)

        self.loop = asyncio.get_event_loop()
        self.loop.run_until_complete(self.start_server)

        def run_forever(loop):
            loop.run_forever()

        thread = threading.Thread(target=run_forever, args=(self.loop,))
        thread.start()

    def stop(self):
        self.loop.stop()

    def konversi_ke_json(self, data):
        return json.dumps(data)

    async def server_websocket(self, websocket, path):
        while True:
            data = self.pengelola_basisdata.get_status_sekarang()
            message = self.konversi_ke_json(data)
            await websocket.send(message)
            await asyncio.sleep(self.config["sediadata"]["websocketdelay"])


import tensorflow as tf
from read_utils import TextConverter, batch_generator,load_origin_data,val_samples_generator
import os
import argparse  # 用于分析输入的超参数

def parseArgs(args):
    """
    Parse 超参数
    Args:
        args (list<stir>): List of arguments.
    """

    parser = argparse.ArgumentParser()
    test_args = parser.add_argument_group('test超参数')
    test_args.add_argument('--file_name', type=str, default='default',help='name of the model')
    test_args.add_argument('--batch_size', type=int, default=100,help='number of seqs in one batch')
    test_args.add_argument('--num_steps', type=int, default=100,help='length of one seq')
    test_args.add_argument('--hidden_size', type=int, default=128,help='size of hidden state of lstm')
    test_args.add_argument('--num_layers', type=int, default=2,help='number of lstm layers')
    test_args.add_argument('--use_embedding', type=bool, default=False,help='whether to use embedding')
    test_args.add_argument('--embedding_size', type=int, default=128,help='size of embedding')
    test_args.add_argument('--learning_rate', type=float, default=0.001,help='learning_rate')
    test_args.add_argument('--train_keep_prob', type=float, default=0.7,help='dropout rate during training')
    test_args.add_argument('--max_steps', type=int, default=100000,help='max steps to train')
    test_args.add_argument('--save_every_n', type=int, default=100,help='save the model every n steps')
    test_args.add_argument('--log_every_n', type=int, default=20,help='log to the screen every n steps')
    test_args.add_argument('--fc_activation', type=str, default='sigmoid', help='funciton of activated')
    test_args.add_argument('--feats', type=str, default='all', help='features of query')
    test_args.add_argument('--batch_norm', type=bool, default=False, help='standardization')
    test_args.add_argument('--op_method', type=str, default='adam', help='method of optimizer')
    test_args.add_argument('--checkpoint_path', type=str, default='models/thoth3/', help='checkpoint path')
    test_args.add_argument('--lr_decay', type=bool, default=False, help='standardization')

    return parser.parse_args(args)

## thoth 问答
args_in = '--file_name n26b200h400F ' \
          '--num_steps 26 ' \
          '--batch_size 200 ' \
          '--learning_rate 0.001 ' \
          '--hidden_size 400 ' \
          '--fc_activation sigmoid ' \
          '--op_method adam ' \
          '--max_steps 200000'.split()

FLAGS = parseArgs(args_in)

def main(_):
    model_path = os.path.join('models', FLAGS.file_name)
    if os.path.exists(model_path) is False:
        os.makedirs(model_path)

    if os.path.exists(os.path.join(model_path, 'converter.pkl')) or os.path.exists(os.path.join(model_path, 'QAs.pkl')) is False:
        print('词库文件不存在,创建...')
        QAs, text = load_origin_data('data/task3_train.txt')
        converter = TextConverter(text, 5000)
        converter.save_to_file(converter.vocab ,os.path.join(model_path, 'converter.pkl'))
        converter.save_to_file(QAs,os.path.join(model_path, 'QAs.pkl'))
    else:
        converter = TextConverter(filename=os.path.join(model_path, 'converter.pkl'))
        QAs = converter.load_obj(filename=os.path.join(model_path, 'QAs.pkl'))

    QA_arrs = converter.QAs_to_arrs(QAs, FLAGS.num_steps)

    thres = int(len(QA_arrs) * 0.9)
    train_samples = QA_arrs[:thres]
    val_samples = QA_arrs[thres:]

    train_g = batch_generator(train_samples, FLAGS.batch_size)
    val_g = val_samples_generator(val_samples)

    print('use embeding:',FLAGS.use_embedding)
    print('vocab size:',converter.vocab_size)

    from model3 import Model
    model = Model(converter.vocab_size,FLAGS,test=False, embeddings=None)

    # 继续上一次模型训练
    FLAGS.checkpoint_path = tf.train.latest_checkpoint(model_path)
    if FLAGS.checkpoint_path:
        model.load(FLAGS.checkpoint_path)

    model.train(train_g,
                FLAGS.max_steps,
                model_path,
                FLAGS.save_every_n,
                FLAGS.log_every_n,
                val_g
                )

if __name__ == '__main__':
    tf.app.run()


import os
import json

def parse(path):
    #file_path = params['training_data_path']
    with open(path , 'r') as f:
        captions = json.load(f)
    return captions

def parse_peer_id(path):
    file = open(path , 'r')
    #done = 0
    ids = []
    for line in file.readlines():
        tmp = line.split('\n')
        ids.append(tmp[0])

    file.close()
    return ids


# en funksjon som finner hvilken plass en gitt karakter har
def find_nth_occurence(search_string,character,n):
    # tom telle variabel
    amount_character = 0
    # en for-løkke som løper gjennom lengden av strengen for å bruke i som index
    for i in range(len(search_string)):
        # sjekker om i index av strengen er lik ønsket karakter
        if character == search_string[i]:
            # legger til en på hvor ofte jeg finner karakteren
            amount_character+=1
            # hvis da mengden er lik antall forekomster n
            # så returnerer den indeksen
            if amount_character == n:
                return i
    # om det ikke skulle være noe forekomst i det hele tatt
    # returnere -1
    return -1


# coding: utf-8

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import pandas as pd

# The hotel_main method extracts the information from the hotels.com website. The method takes in the
# string of the city name, check in date, and check out date, and uses the selenium with chrome driver
# and beautifulsoup to extract information from the website.
def hotel_main(city, checkInDate, checkOutDate):
    url = "https://www.hotels.com/"
    urlsAndCities = set()
    completeUrl = url
    completeUrl += "search.do?q-destination=" + city + "&q-check-in=" + checkInDate + "&q-check-out=" + checkOutDate

    # Open the chrome driver, scroll down using selenium and get the whole html page; find all the hotel tags
    driver = webdriver.Chrome(executable_path="./chromedriver")
    driver.get(completeUrl)
    body = driver.find_element_by_css_selector('body')
    for num in range(400):
        body.send_keys(Keys.PAGE_DOWN)
    html = driver.execute_script("return document.documentElement.outerHTML")
    soup = BeautifulSoup(html, features="html.parser")
    hotels = soup.find_all(class_="hotel-wrap")
    result = {}

    # for each hotel html part, extract information from the block
    for hotel in hotels:
        name = hotel.find(class_="p-name").text.strip()
        street = hotel.find(class_="contact").text.strip()[:-14]
        phone = hotel.find(class_="contact").text.strip()[-12:]
        try:
            guest_reviews = hotel.find(class_="guest-reviews-badge").text.strip()[-3:]
            guest_reviews = str(float(guest_reviews) / 10 * 5)

        except:
            guest_reviews = None
        try:
            popularity = hotel.find(class_="small-view").text.strip()[:-8].replace(',', '')
        except:
            popularity = None
        try:
            price = hotel.find(class_="price").text.strip()
            digits = [int(s) for s in price.split("$") if s.isdigit()]
            price = digits[-1]
        except:
            price = None

        result[name] = [name, street, guest_reviews, popularity, price, "", "", "", phone, ""]

    # Construct the dataframe from the combined dictionary in the indexed order
    columns = ["Name", "address", "rating", "popularity", "price", "category", "url", "imageurl", "phone",
               "description"]
    df = pd.DataFrame.from_dict(data=result, orient="index", columns=columns)
    return df

if __name__ == '__main__':
    hotel_main("Pittsburgh", "2019-03-01", "2019-03-02")


import neuralnet as nn
import numpy as np
from sklearn import datasets
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
import matplotlib.pyplot as plt
import seaborn as sns

#Load the sklearn handwritten digits dataset
digits = datasets.load_digits()
X = digits.images.reshape((len(digits.images), -1))
labels = digits.target

#Normalize the data between 0 and 1
normalizer = MinMaxScaler()
X = normalizer.fit_transform(X)

#One-Hot Encode the output
lr = np.arange(10)
labels = [(k == lr).astype(int) for k in labels]
labels = np.vstack(labels)

#Split data
X_train, X_test, y_train, y_test = train_test_split(X, labels)

#Create a simple MLP
net = nn.Net(input_size = 64, lr = 0.1, momentum = 0.1, loss = 'CategoricalCrossEntropy')
net.add_layer(512)
net.add_layer(256)
net.add_layer(10, activation = 'sigmoid')

#Train the network using the training data
net.fit(X_train, y_train, batch_size = 1, epochs = 200, min_delta = 0.01, patience = 3)

#Predict the class of the test images
pred = net.predict(X_test)

#Go from one-hot encoding back to integers between 0 and 9
y_pred = [np.argmax(r) for r in pred]
y_true = [np.argmax(r) for r in y_test]

#Compute the accuracy of the network on the test data
print(accuracy_score(y_true, y_pred))
conf_matrix = confusion_matrix(y_true, y_pred)
sns.heatmap(conf_matrix, annot = True)
plt.xlabel('Predicted')
plt.ylabel('True')


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import pandas as pd
import numpy as np
import datetime
from datetime import timedelta
import plotly.figure_factory as ff
import dash_table
import itertools
import CreateSchedules
import os
import re

#Create the schedules...
CreateSchedules.CheckSchedules()
#extract all the csv file name in schedule folder
Schedules_avail = [ f.name for f in os.scandir('Schedules') if f.is_file() & bool(re.search('.*\.csv$', f.name))]

#Extract the name of the schedule
Schedules_avail = [x.replace('_Schedule.csv', '') for x in Schedules_avail]

#create options for dropdown
schedulesDict = [{'label' : i, 'value': i} for i in Schedules_avail]

app = dash.Dash(__name__)

app.layout = html.Div(
    children = [html.H1('Movie Schedules for I20 cinemas'),

                #dropdown
        html.P(children=[
        html.Label('Select a Schedule'),
        dcc.Dropdown(id='select', options= schedulesDict, multi = False, value = "TB_Original")],
            style = {'width':'400px'}),

        html.Div(children = [
            html.P("C: Cleaning time, T: Trailer time, P: Pre-show advertisement"),
            dcc.Graph(id = 'gantt'),
            html.P("Type in a movie or time slot to filter your choices:"),
            dash_table.DataTable( id='showtime',
            columns=[{'name' : 'Movie', 'id' : 'movie'},
                     {'name' : 'Theatre', 'id' : 'theatre'},
                     {'name' : 'Start Time', 'id' : 'startTimeDate'}],
            sort_action= "native",
            sort_mode="multi",
            filter_action="native",
            row_selectable="single"),
        html.A('Download Data',
        id='download-link',
        href="",
        target="_blank"
    )
                ]) #closes out div for graph and data table
    ]) #closes out outer div

@app.callback(
        [Output('gantt', 'figure'),
        Output('showtime','data')],
        [Input('select', 'value')])

def update_output(select):

    startTimesDF = pd.read_table('Schedules/' + select + '_Schedule.csv', delimiter = ',', header = 'infer')
    startTimesDF = startTimesDF.sort_values(by=['theatre'],ascending=False)

    #I will generate pre and post show time data, I manually compute/input time unit and duration for each of these actitives here.
    #generate cleaning data
    #cleaning starts when movie ends
    cleaning_data = startTimesDF[['theatre','endTimeDate']]
    #change the name so later I can concatenate the data
    cleaning_data.columns = ['theatre','startTimeDate']
    #create time unit, make it 0 so this column is not missing for cleaning time
    cleaning_data['timeUnit'] = 0
    #check to see if this block is a movie
    cleaning_data['check'] = 'N'
    #compute endtime for cleanings, which is 15 mins
    cleaning_data['endTimeDate'] = pd.to_datetime(cleaning_data['startTimeDate']) + timedelta(minutes=15)
    #create title for those time slots on Gantt chart
    cleaning_data['movie'] = 'C'

    #generate trailer data
    #Trailer ends when movies starts
    trailer_data = startTimesDF[['theatre','startTimeDate']]
    #change the name of columns
    trailer_data.columns = ['theatre','endTimeDate']
    #create time unit, same logic as cleaning time
    trailer_data['timeUnit'] = 0
    trailer_data['check'] = 'N'    #check to see if this block is a movie
    #compute start time for trailer, which is 20 min before the movie starts
    trailer_data['startTimeDate'] = pd.to_datetime(trailer_data['endTimeDate']) - timedelta(minutes=20)
    #create title for those time slots on Gantt chart
    trailer_data['movie'] = 'T'

    #generate preshow_data
    #preshow ends when trailer starts
    preshow_data = trailer_data[['theatre','startTimeDate']]
    #change the name of columns
    preshow_data.columns = ['theatre','endTimeDate']
    #create time unit same logic as above
    preshow_data['timeUnit'] = 0
    preshow_data['check'] = 'N'    #check to see if this block is a movie

    #compute start time for trailer, which is 30 min before the trailer starts
    preshow_data['startTimeDate'] = pd.to_datetime(preshow_data['endTimeDate']) - timedelta(minutes=30)
    #create title for those time slots on Gantt chart
    preshow_data['movie'] = 'P'

    startTimesDF_chart = pd.concat([startTimesDF,cleaning_data, trailer_data,preshow_data])

    #check column used to check if the time slot is a movie or not, if not a movie, the annotated time is 5 mins after start time, if it is a movie, then the annotation will be have x axis at 50 mins after the movie starts. I have to do this manually and have not found a way to to position the annotation nicely
        #create columns to position the label
    startTimesDF_chart['AnnotatedTime'] =  np.where(startTimesDF_chart['check'] == 'N', pd.to_datetime(startTimesDF_chart['startTimeDate']) + timedelta(minutes=5), pd.to_datetime(startTimesDF_chart['startTimeDate']) + timedelta(minutes=50))

    #copied from LP project, used to create dictionaries for plot

    showit = [dict(Task= "Theater" + str(row[1].theatre), Start= row[1].startTimeDate, Finish = row[1].endTimeDate, Resource = row[1].movie) for row in startTimesDF_chart.iterrows()]

    #colors = dict(T = 'red', C = 'blue', P = 'black')
    #unique_movie = np.unique(startTimesDF.movie)

    #colors2 = dict((movie, np.random.rand(3,)) for movie in unique_movie)
    #colors = colors.update(colors2),
    fig = ff.create_gantt(showit, index_col='Resource', group_tasks=True, show_hover_fill = True,show_colorbar=True,showgrid_x=True)

    #create dictionnary for info for the annotation
    annots = [dict(x=row[1].AnnotatedTime,y=row[1].theatre-1,text=row[1].movie, showarrow=False, font=dict(color='white')) for row in startTimesDF_chart.iterrows() ]
    #[dict(x=row.startTimeDate,y=row.theater,text=row.movie, showarrow=False, font=dict(color='white'))]

    fig['layout']['annotations'] = annots

    return fig, startTimesDF.to_dict('records')

@app.callback(
    Output('download-link', 'href'),
    [Input('select', 'value')])
def update_download_link(select):
    startTimesDF = pd.read_table('Schedules/' + select + '_Schedule.csv', delimiter = ',', header = 'infer')

    csv_string = startTimesDF.to_csv()
    return csv_string

if __name__ == "__main__":
    app.run_server(debug = True)


import threading
from queue import Queue
from spider import Spider
from domain import *
from general import *

PROJECT_NAME = "book"
HOMEPAGE = "https://books2read.com/"
DOMAIN_NAME = getDomainName(HOMEPAGE)
QUEUE_FILE = PROJECT_NAME + "/queue.txt"
CRAWLED_FILE = PROJECT_NAME + "/crawled.txt"
NUMBER_OF_THREADS = 8
queue = Queue()
Spider(PROJECT_NAME, HOMEPAGE, DOMAIN_NAME)

# Create worker threads (will die when main exists)
def createWorkers():
    for _ in range(NUMBER_OF_THREADS):
        t = threading.Thread(target=work)
        t.daemon = True
        t.start()

# Do the next job in the queue
def work():
    while True:
        url = queue.get()
        Spider.crawlPage(threading.current_thread().name, url)
        queue.task_done()

# Each queued links is a new job
def createJobs():
    for link in fileToSet(QUEUE_FILE):
        queue.put(link)
    queue.join()
    crawl()

# Check if there are items in the queue, if so crawl them
def crawl():
    queuedLinks = fileToSet(QUEUE_FILE)
    if len(queuedLinks) > 0:
        print(str(len(queuedLinks)) + " links in the queue")
        createJobs()

createWorkers()
crawl()


import os
from tempfile import TemporaryDirectory

from kaos_backend.clients.pachyderm import PachydermClient
from kaos_backend.services.job_service import JobService
from kaos_backend.util.tests import create_zip_file
from kaos_backend.util.validators import BundleValidator
from python_pachyderm import PpsClient, PfsClient

def t_any(cls):
    """
        Verifies if input is of same class
    """
    class AnyClass(cls):
        def __eq__(self, other):
            return True

    return AnyClass()

def create_job_service(mocker, workspaces=None):
    """
    Creates a Mock job Service

    Return:
         service: `kaos_backend.services.JobService`

    """
    client = PachydermClient(PpsClient(), PfsClient())
    service = JobService(client)

    def check_pipeline_exists_mock(pipeline_name):
        return pipeline_name.split('-')[-1] in workspaces

    client.check_pipeline_exists = mocker.Mock(side_effect=check_pipeline_exists_mock)

    return service

def create_test_file(dirname, filename):
    f = open(os.path.join(dirname, filename), "w")
    f.write("this is fake")
    f.close()
    return f

def create_bundle_zip(files):
    tempdir = TemporaryDirectory()
    root_dir = tempdir.name

    main_dir = os.path.join(root_dir, "test")
    os.mkdir(main_dir)

    model_dir = os.path.join(main_dir, BundleValidator.MODEL)
    os.mkdir(model_dir)

    filename = "Dockerfile"
    create_test_file(main_dir, filename)

    for f in files:
        create_test_file(model_dir, f)

    zip_filename = create_zip_file(root_dir)
    with open(zip_filename, 'rb') as file_data:
        bytes_content = file_data.read()

    return bytes_content, tempdir, zip_filename

def create_train_zip():
    return create_bundle_zip(BundleValidator.REQUIRED_TRAINING_FILES)

def create_inference_zip():
    return create_bundle_zip(BundleValidator.REQUIRED_INFERENCE_FILES)


nota01 = float(input('Digite a primeira nota: '))
nota02 = float(input('Digite a segunda nota: '))
nota03 = float(input('Digite a terceira nota: '))

def media(nota01, nota02, nota03):
    if nota01 < 0 or nota01 > 10:
        print('Nota 01 inválida')

    else:
        if nota02 < 0 or nota02 > 10:
            print('Nota 02 inválida')

        else:
            if nota03 < 0 or nota03 > 10:
                print('Nota 03 inválida')

            else:
                total = ((nota01 * 4) + (nota02 * 4) + (nota03 * 2)) / 10
                print('A média ponderada é {:.2f}'.format(total))

                return total

media_notas = media(nota01, nota02, nota03)

def validar_media(media_notas):
    if media_notas < 0 or media_notas > 10:
        print('Media invalida')
    else:
        if media_notas > 5.9:
            print('Aprovado')
        else:
            print('Reprovado')

def maior_nota(nota01, nota02, nota03):
    if nota01 > nota02 and nota01 > nota03:
        print('Sua maior nota foi a nota 01: {}'.format(nota01))
    else:
        if nota02 > nota01 and nota02 > nota03:
            print('Sua maior nota foi a nota 02: {}'.format(nota02))
        else:
            print('Sua maior nota foi a nota 03: {}'.format(nota03))

validar_media(media_notas)
maior_nota(nota01, nota02, nota03)