from safegtk import SafeGTK
import rpyc
import gobject
import gtk
import pygtk
pygtk.require('2.0')

def BrowserServiceFactory(browser):
    class BrowserService(rpyc.Service):
        def on_connect(self, conn):
            conn._config["allow_public_attrs"] = True

        def exposed_navigate(self, url):
            old_url = browser.txt_url.get_text()
            if url.startswith("/"):
                base = old_url.split("/")[0]
                url = base + url
            browser.txt_url.set_text(url)
            browser.on_navigate(None)
    return BrowserService

class Browser(object):
    def __init__(self):
        self.conn = None

        window = gtk.Window(gtk.WINDOW_TOPLEVEL)
        window.set_resizable(True)
        window.connect("destroy", self.on_close)
        window.set_title("Web8 Browser")
        self.box_main = gtk.VBox()
        self.box_main.show()
        window.add(self.box_main)

        hbox1 = gtk.HBox()
        hbox1.show()
        self.box_main.pack_start(hbox1, fill=False, expand=False)

        self.txt_url = gtk.Entry()
        self.txt_url.set_text("localhost/main")
        self.txt_url.show()
        self.txt_url.connect("activate", self.on_navigate)
        hbox1.pack_start(self.txt_url)

        btn_send = gtk.Button("Go")
        btn_send.show()
        btn_send.connect("clicked", self.on_navigate)
        hbox1.pack_start(btn_send, fill=False, expand=False, padding=10)

        self.box_content = None
        window.show()

    def on_close(self, widget):
        if self.conn:
            self.conn.close()
            self.conn = None
        gtk.main_quit()

    def on_navigate(self, widget, data=None):
        url = self.txt_url.get_text()
        if "/" not in url:
            url += "/"
        host, page = url.split("/", 1)
        if ":" in host:
            addr, port = host.split(":", 1)
            port = int(port)
        else:
            addr = host
            port = 18833
        if self.conn:
            self.conn.close()
            self.conn = None

        if self.box_content:
            self.box_main.remove(self.box_content)
            self.box_content.destroy()
            self.box_content = None
        self.box_content = gtk.VBox()
        self.box_content.show()
        self.box_main.pack_start(self.box_content)

        self.conn = rpyc.connect(host, port, service=BrowserServiceFactory(self))
        gobject.io_add_watch(self.conn, gobject.IO_IN, self.bg_server)
        self.conn.root.get_page(SafeGTK, self.box_content, page)

    def bg_server(self, source=None, cond=None):
        if self.conn:
            self.conn.poll_all()
            return True
        else:
            return False

if __name__ == "__main__":
    b = Browser()
    gtk.main()


#!/usr/bin/env python3

import sys, os, csv, itertools, pickle, lzma, json, hashlib, tempfile, difflib, re, glob
from pprint import pprint, pformat
from collections import namedtuple
from counter import Ticket, PreferenceFlow, PapersForCount, SenateCounter

def named_tuple_iter(name, reader, header, **kwargs):
    field_names = [ t for t in [t.strip().replace('-', '_') for t in header] if t ]
    typ = namedtuple(name, field_names)
    mappings = []
    for field_name in kwargs:
        idx = field_names.index(field_name)
        mappings.append((idx, kwargs[field_name]))
    for row in reader:
        for idx, map_fn in mappings:
            row[idx] = map_fn(row[idx])
        yield typ(*row)

def int_or_none(s):
    try:
        return int(s)
    except ValueError:
        return None

class Candidates:
    def __init__(self, candidates_csv):
        self.by_id = {}
        self.by_name_party = {}
        self.candidates = []
        self.load(candidates_csv)

    def load(self, candidates_csv):
        with lzma.open(candidates_csv, 'rt') as fd:
            reader = csv.reader(fd)
            version = next(reader)
            header = next(reader)
            for candidate in named_tuple_iter('Candidate', reader, header, CandidateID=int):
                self.candidates.append(candidate)
                assert(candidate.CandidateID not in self.by_id)
                self.by_id[candidate.CandidateID] = candidate
                k = (candidate.Surname, candidate.GivenNm, candidate.PartyNm)
                assert(k not in self.by_name_party)
                self.by_name_party[k] = candidate

    def lookup_id(self, candidate_id):
        return self.by_id[candidate_id]

    def lookup_name_party(self, surname, given_name, party):
        return self.by_name_party[(surname, given_name, party)]

    def get_parties(self):
        return dict((t.PartyAb, t.PartyNm) for t in self.candidates if t)

class SenateATL:
    def __init__(self, state_name, candidates, gvt_csv, firstprefs_csv):
        self.gvt = {}
        self.ticket_votes = []
        self.individual_candidate_ids =[]
        self.candidate_order = {}
        self.candidate_title = {}
        self.btl_firstprefs = {}
        self.load_tickets(candidates, gvt_csv)
        self.load_first_preferences(state_name, firstprefs_csv)

    def load_tickets(self, candidates, gvt_csv):
        with lzma.open(gvt_csv, 'rt') as fd:
            reader = csv.reader(fd)
            header = next(reader)
            it = sorted(named_tuple_iter('GvtRow', reader, header, Preference=int, TicketNo=int, Ticket=lambda t: t.strip()), key=lambda gvt: (gvt.StateAb, gvt.Ticket, gvt.TicketNo, gvt.Preference))
            for (state_ab, ticket, ticket_no), g in itertools.groupby(it, lambda gvt: (gvt.StateAb, gvt.Ticket, gvt.TicketNo)):
                prefs = []
                for ticket_entry in g:
                    candidate = candidates.lookup_name_party(ticket_entry.Surname, ticket_entry.GivenName, ticket_entry.Party)
                    prefs.append((ticket_entry.Preference, candidate.CandidateID))
                if ticket not in self.gvt:
                    self.gvt[ticket] = []
                self.gvt[ticket].append(PreferenceFlow(tuple(prefs)))

    def load_first_preferences(self, state_name, firstprefs_csv):
        with lzma.open(firstprefs_csv, 'rt') as fd:
            reader = csv.reader(fd)
            version = next(reader)
            header = next(reader)
            for idx, row in enumerate(named_tuple_iter('StateFirstPrefs', reader, header, TotalVotes=int, CandidateID=int)):
                if row.StateAb == state_name:
                    if row.CandidateDetails.endswith('Ticket Votes'):
                        self.ticket_votes.append((row.Ticket, row.TotalVotes))
                    elif row.CandidateDetails != 'Unapportioned':
                        assert(row.CandidateID not in self.btl_firstprefs)
                        self.btl_firstprefs[row.CandidateID] = row.TotalVotes
                        self.individual_candidate_ids.append(row.CandidateID)
                self.candidate_order[row.CandidateID] = idx
                self.candidate_title[row.CandidateID] = row.CandidateDetails

    def get_tickets(self):
        for group, n in self.ticket_votes:
            yield Ticket(tuple(self.gvt[group])), n

    def get_candidate_ids(self):
        return self.individual_candidate_ids

    def get_candidate_order(self, candidate_id):
        return self.candidate_order[candidate_id]

    def get_candidate_title(self, candidate_id):
        return self.candidate_title[candidate_id]

class SenateBTL:
    """
    build up Ticket instances for each below-the-line vote in the AEC dataset
    aggregate BTL votes with the same preference flow together
    """

    def __init__(self, candidates, btl_csv):
        self.ticket_votes = {}
        self.load_btl(candidates, btl_csv)

    def load_btl(self, candidates, btl_csv):
        with lzma.open(btl_csv, 'rt') as fd:
            reader = csv.reader(fd)
            version = next(reader)
            header = next(reader)
            it = named_tuple_iter('BtlRow', reader, header, Batch=int, Paper=int, Preference=int_or_none, CandidateId=int)
            for key, g in itertools.groupby(it, lambda r: (r.Batch, r.Paper)):
                ticket_data = []
                for row in sorted(g, key=lambda x: x.CandidateId):
                    ticket_data.append((row.Preference, row.CandidateId))
                ticket = Ticket((PreferenceFlow(ticket_data),))
                if ticket not in self.ticket_votes:
                    self.ticket_votes[ticket] = 0
                self.ticket_votes[ticket] += 1

    def get_tickets(self):
        for ticket in sorted(self.ticket_votes, key=lambda x: self.ticket_votes[x]):
            yield ticket, self.ticket_votes[ticket]

def senate_count(fname, state_name, vacancies, data_dir, *args, **kwargs):
    def load_tickets(ticket_obj):
        if ticket_obj is None:
            return
        for ticket, n in ticket_obj.get_tickets():
            tickets_for_count.add_ticket(ticket, n)
    df = lambda x: glob.glob(os.path.join(data_dir, x))[0]
    candidates = Candidates(df('SenateCandidatesDownload-*.csv.xz'))
    atl = SenateATL(state_name, candidates, df('wa-gvt.csv.xz'), df('SenateFirstPrefsByStateByVoteTypeDownload-*.csv.xz'))
    btl = SenateBTL(candidates, df('SenateStateBTLPreferences-*-WA.csv.xz'))
    tickets_for_count = PapersForCount()
    load_tickets(atl)
    load_tickets(btl)
    counter = SenateCounter(
        fname,
        vacancies,
        tickets_for_count,
        candidates.get_parties(),
        atl.get_candidate_ids(),
        lambda candidate_id: atl.get_candidate_order(candidate_id),
        lambda candidate_id: atl.get_candidate_title(candidate_id),
        lambda candidate_id: candidates.lookup_id(candidate_id).PartyAb,
        *args,
        **kwargs)

def verify_test_logs(verified_dir, test_log_dir):
    test_re = re.compile(r'^round_(\d+)\.json')
    rounds = []
    for fname in os.listdir(verified_dir):
        m = test_re.match(fname)
        if m:
            rounds.append(int(m.groups()[0]))
    def fname(d, r):
        return os.path.join(d, 'round_%d.json' % r)
    def getlog(d, r):
        with open(fname(d, r)) as fd:
            return json.load(fd)
    ok = True
    for idx in sorted(rounds):
        v = getlog(verified_dir, idx)
        t = getlog(test_log_dir, idx)
        if v != t:
            print("Round %d: FAIL" % idx)
            print("Log should be:")
            pprint(v)
            print("Log is:")
            pprint(t)
            print("Diff:")
            print('\n'.join(difflib.unified_diff(pformat(v).split('\n'), pformat(t).split('\n'))))
            ok = False
        else:
            print("Round %d: OK" % idx)
    if ok:
        for fname in os.listdir(test_log_dir):
            if test_re.match(fname):
                os.unlink(os.path.join(test_log_dir, fname))
        os.rmdir(test_log_dir)
    return ok

def main():
    config_file = sys.argv[1]
    out_dir = sys.argv[2]
    base_dir = os.path.dirname(os.path.abspath(config_file))
    with open(config_file) as fd:
        config = json.load(fd)
    json_f = os.path.join(out_dir, 'count.json')
    with open(json_f, 'w') as fd:
        obj = {
            'house': config['house'],
            'state' : config['state']
        }
        obj['counts'] = [{
            'name' : count['name'],
            'description' : count['description'],
            'path' : count['shortname']}
            for count in config['count']]
        json.dump(obj, fd)

    test_logs_okay = True
    for count in config['count']:
        data_dir = os.path.join(base_dir, count['path'])
        test_log_dir = None
        if 'verified' in count:
            test_log_dir = tempfile.mkdtemp(prefix='dividebatur_tmp')
        outf = os.path.join(out_dir, count['shortname'] + '.json')
        print("counting %s -> %s" % (count['name'], outf))
        senate_count(
            outf,
            config['state'], config['vacancies'], data_dir,
            count.get('automation') or [],
            test_log_dir,
            name=count.get('name'),
            description=count.get('description'),
            house=config['house'],
            state=config['state'])
        if test_log_dir is not None:
            if not verify_test_logs(os.path.join(base_dir, count['verified']), test_log_dir):
                test_logs_okay = False
    if not test_logs_okay:
        print("** TESTS FAILED **")
        sys.exit(1)

if __name__ == '__main__':
    main()


import cv2
from djitellopy import Tello
import numpy as np
import torch
from torch import nn
import torchvision

from create_apple_model import Categories as CAP_Categories
from create_rotten_model import Categories as CRM_Categories
from predict_ripe import RipenessPredictor

class CropToSquare:
    def __call__(self, img: torch.Tensor):
        h, w = img.shape[-2:]
        if h > w:
            diff = h - w
            img = img[..., diff // 2 : -diff // 2, :]
        elif w > h:
            diff = w - h
            img = img[..., :, diff // 2 : -diff // 2]
        return img

def detect_apple(model_apple_in_view: nn.Module, image: torch.Tensor, device) -> bool:
    image = image.to(device).unsqueeze(0)
    image = CropToSquare()(image)
    out = model_apple_in_view(image)
    return torch.argmax(out).item() == CAP_Categories.APPLE.value

def detect_rotten(model_detect_rotten: nn.Module, image, device):
    image = image.to(device).unsqueeze(0)
    image = CropToSquare()(image)
    out = model_detect_rotten(image)
    return torch.argmax(out).item() == CRM_Categories.ROTTEN.value

def detect_ripe(model_ripeness_predictor: RipenessPredictor, image):
    return model_ripeness_predictor.predict_ripe(image)

def find_apple_center(drone, model_haar_cascade, image):
    apples = model_haar_cascade.detectMultiScale(image)
    marked_image = image.copy()
    apple_centers = []
    apple_areas = []

    for (x, y, w, h) in apples:
        cv2.rectangle(marked_image, (x, y), (x + w, y + h), (0, 0, 255), 2)
        # get centerx and centery values
        cx = x + w // 2
        cy = y + h // 2
        # calculate the total area
        area = w * h
        # draw a circle where the center is
        cv2.circle(marked_image, (cx, cy), 5, (0, 255, 0), cv2.FILLED)
        # append these values to the lists we created above
        apple_centers.append([cx, cy])
        apple_areas.append(area)
    # get the index of the max area detected over coming frame
    # so that focus on the bigger face if there are multiple faces
    if apple_areas:
        i = apple_areas.index(max(apple_areas))
        return marked_image, [apple_centers[i], apple_areas[i]]
    else:
        return marked_image, [[0, 0], 0]

def track_apple(drone, info, w, pid, p_error, area_range):
    forward_vel = 0
    area = info[1]  # area value
    x, y = info[0]  # centerx and centery values

    error = x - w // 2  # what is the difference between our object and the center
    yaw_vel = pid[0] * error + pid[1] * (
        error - p_error
    )  # calculate yaw value with using PID method
    yaw_vel = int(np.clip(yaw_vel, -100, 100))

    if (
        area_range[0] < area < area_range[1]
    ):  # if the object is in the range we indicated, do not move
        forward_vel = 0
    elif area > area_range[1]:  # if it is too close, move backward
        forward_vel = -20
    elif (
        area < area_range[0] and area != 0
    ):  # if it is too far and if it detects an apple, move forward
        forward_vel = 20

    if (
        x == 0
    ):  # if it does not detect an apple, both speed and error should be 0 so the drone
        # will not rotate
        yaw_vel = 0
        error = 0

    drone.send_rc_control(0, forward_vel, 0, yaw_vel)
    return error

def fruit_fly(drone: Tello):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    (
        model_apple_in_view,
        model_haar_cascade,
        model_detect_rotten,
        model_ripeness_predictor,
    ) = load_models(device)

    drone.connect()
    drone.LOGGER.info(f"Battery: {drone.get_battery()}%")

    img_to_tensor = torchvision.transforms.ToTensor()
    error = 0
    rdi_size = (100, 100)
    try:
        while True:
            img = drone.get_frame_read().frame

            # Convert the image to PyTorch tensor with the appropriate coloring
            torch_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
            torch_img = img_to_tensor(torch_img)
            apple_detected = (
                detect_apple(model_apple_in_view, torch_img, device)
            )
            if apple_detected:
                # Move towards apple
                marked_image, info = find_apple_center(drone, model_haar_cascade, img)
                error = track_apple(
                    drone, info, img.shape[1], [0.4, 0.4, 0], error, [6200, 6800]
                )

                # Trim image to only include the detected apple
                center_x, center_y = info[0]
                area = info[1]
                if area != 0:
                    side_length = np.sqrt(area)
                    x1 = int(center_x - side_length // 2)
                    x2 = int(center_x + side_length // 2)
                    y1 = int(center_y - side_length // 2)
                    y2 = int(center_y + side_length // 2)
                    torch_img = torch_img[..., y1:y2, x1:x2]
                    trimmed_image = cv2.resize(img[y1:y2, x1:x2], rdi_size)

                    # Detect ripeness/rotten
                    is_rotten = detect_rotten(model_detect_rotten, torch_img, device)
                    is_ripe = detect_ripe(model_ripeness_predictor, trimmed_image)

                    print(f"Rotten: {is_rotten}, Ripe: {is_ripe}")
                    cv2.imshow("Ripeness Detector Image", trimmed_image)

                img = marked_image

            cv2.imshow("Output", img)
            # if we click 'q' then land the drone
            if cv2.waitKey(1) & 0xFF == ord("q"):
                break
    finally:
        drone.land()
        drone.streamoff()

def load_mobilenet(
    num_outputs: int,
    device: str | torch.device = None,
):
    if device is None:
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = torch.hub.load("pytorch/vision:v0.10.0", "mobilenet_v2", verbose=False)
    # Replace the last layer with the number of classes we want
    model.classifier[-1] = nn.Linear(model.classifier[-1].in_features, num_outputs)
    model.to(device)
    return model

def load_models(
    device,
) -> tuple[nn.Module, cv2.CascadeClassifier, nn.Module, RipenessPredictor]:
    apple_in_view = load_mobilenet(num_outputs=2).to(device)
    apple_in_view.load_state_dict(
        torch.load("models/detect_apple/all_files_16its_2022-11-22_18-33-45.pt")
    )
    apple_in_view.eval()

    haar_cascade = cv2.CascadeClassifier("models/cascade/curated_set_cascade.xml")
    detect_rotten = load_mobilenet(num_outputs=2).to(device)
    detect_rotten.load_state_dict(
        torch.load("models/detect_rotten/all_files_16its_2022-11-23_13-55-28.pt")
    )
    detect_rotten.eval()

    ripeness_predictor = RipenessPredictor("models/ripeness/ripeness_model.sav")

    return (
        apple_in_view,
        haar_cascade,
        detect_rotten,
        ripeness_predictor,
    )

class WebcamTello(Tello):
    def __init__(self):
        import djitellopy

        Tello.threads_initialized = True
        host = "192.168.0.1.WEBCAM"
        super().__init__(host)
        self.responses = djitellopy.tello.drones[host]["responses"]
        self.state = djitellopy.tello.drones[host]["state"]
        self.responses.append("DUMMY")
        self.state["DUMMY"] = "DUMMY"

    def get_udp_video_address(self):
        return 0

    def send_command_with_return(
        self, command: str, timeout: int = Tello.RESPONSE_TIMEOUT
    ) -> str:
        return "okay"

    def send_command_without_return(self, command: str) -> None:
        pass

    def get_state_field(self, key: str):
        self.state[key] = -1
        return super().get_state_field(key)

def main():
    # Swap this line to use the real drone
    # drone = Tello()
    drone = WebcamTello()
    fruit_fly(drone)

if __name__ == "__main__":
    main()


import streamlit as st
import numpy as np
import pandas as pd
import plotly.express as px
import ast

def read_df():
    df = pd.read_csv("250_top_IMDB.csv")

    def convert_runtime(runtime):
        parts = runtime.split()
        total_minutes = 0
        for part in parts:
            if "h" in part:
                total_minutes += int(part.replace("h", "")) * 60
            elif "m" in part:
                total_minutes += int(part.replace("m", ""))
        return total_minutes

    df["runtime"] = df["runtime"].apply(convert_runtime)
    nanValue = df["gross_us_canada"][4]

    def convert_gross(gross_val):
        if nanValue is gross_val:
            return 0
        else:
            return int(gross_val.replace("$", "").replace(",", ""))

    def convert_parental_guide(parental_guide_val):
        if (
            parental_guide_val in ["null", "blank", "Not Rated"]
            or nanValue is parental_guide_val
        ):
            return "Unrated"
        return parental_guide_val

    def extract_keys(dictionary):
        return list(dictionary.keys())[0]

    df["gross_us_canada"] = df["gross_us_canada"].apply(convert_gross)
    df["parental_guide"] = df["parental_guide"].apply(convert_parental_guide)

    df["genre"] = df["genre"].apply(ast.literal_eval)
    df["directore"] = df["directore"].apply(ast.literal_eval)
    df["writer"] = df["writer"].apply(ast.literal_eval)
    df["star"] = df["star"].apply(ast.literal_eval)

    df["dict_id"] = df["dict_id"].apply(ast.literal_eval)
    df["dict_id"] = df["dict_id"].apply(extract_keys)

    df.drop(columns=["storyline"], inplace=True)

    return df

def top_stars(df, top_n=10):
    df_star = pd.DataFrame(
        columns=[
            "person_id",
            "person_name",
        ]
    )

    def create_df_main(df, df_star):
        for index, row in df.iterrows():
            star = row["star"]
            for i in star:
                new_row = {
                    "person_id": star[i],
                    "person_name": i,
                }
                df_star.loc[len(df_star)] = new_row
        return df_star

    df_star = create_df_main(df, df_star)
    temp = df_star.value_counts()
    temp = temp.head(top_n)
    return pd.DataFrame(temp).reset_index()

df = read_df()
top_n = 10

st.set_page_config(page_title="Static Charts")
st.title("Second Subcategory; Static Charts")

top_10_movies = df.nlargest(top_n, "gross_us_canada")
fig = px.bar(
    top_10_movies,
    x="title",
    y="gross_us_canada",
    title="Bar Chart of 10 Best Selling Movies",
)
tab1, tab2 = st.tabs(["Streamlit theme (default)", "Plotly native theme"])
with tab1:
    st.plotly_chart(fig, theme="streamlit")
with tab2:
    st.plotly_chart(fig, theme=None)

top_10_stars = top_stars(df)
fig = px.bar(
    top_10_stars,
    x="person_name",
    y="count",
    title="Bar Chart of the 10 Most Prolific Actors",
)
tab1, tab2 = st.tabs(["Streamlit theme (default)", "Plotly native theme"])
with tab1:
    st.plotly_chart(fig, theme="streamlit")
with tab2:
    st.plotly_chart(fig, theme=None)

temp = df.explode("genre")
temp = temp["genre"].value_counts()
temp = pd.DataFrame(temp).reset_index()
fig = px.pie(
    temp, values="count", names="genre", title="Pie Chart of the number of each Genre"
)
tab1, tab2 = st.tabs(["Streamlit theme (default)", "Plotly native theme"])
with tab1:
    st.plotly_chart(fig, theme="streamlit")
with tab2:
    st.plotly_chart(fig, theme=None)

temp = df["parental_guide"].value_counts()
temp = pd.DataFrame(temp).reset_index()
fig = px.pie(
    temp,
    values="count",
    names="parental_guide",
    title="Pie Chart of the number of `parental_guide` Rating in Movies",
)
tab1, tab2 = st.tabs(["Streamlit theme (default)", "Plotly native theme"])
with tab1:
    st.plotly_chart(fig, theme="streamlit")
with tab2:
    st.plotly_chart(fig, theme=None)

df_exploded = df.explode("genre")
genre_age_counts = (
    df_exploded.groupby(["genre", "parental_guide"]).size().unstack(fill_value=0)
)
genre_age_counts
st.bar_chart(genre_age_counts)


import csv
import os
import re
#from math import round
import sys
import numpy
from matplotlib.lines import Line2D
import matplotlib.pyplot as plt

plt.rcParams.update({
    "text.usetex": True,
    "font.family": "Helvetica",
    "font.size": 18
})

if len(sys.argv)==2:
    file='-'+sys.argv[1]
else:
    file=""

if "DISPLAY" not in os.environ:
    os.environ["DISPLAY"] = ":0"

xdata = {}
ydata = {}

name_to_nx = {}
name_to_nt = {}

with open('perfdata'+file+'.csv','r',newline='') as cfd:
    skip = True
    for row in csv.reader(cfd):
        if skip:
            skip = False
            continue
        #name = "%s-%d" % (row[0], int(row[1]))
        name = row[0]
        nx = row[1]
        nt = row[2]
        if name in name_to_nt:
            if name_to_nt[name] != nt:
                print("WARNING: Multiple values of nt were used")
        else:
            name_to_nt[name] = nt
        if name in name_to_nx:
            if name_to_nx[name] != nx:
                print("WARNING: Multiple values of nx were used")
        else:
            name_to_nx[name] = nx
        if name not in xdata:
            xdata[name] = []
            ydata[name] = []
        nthreads = float(row[3])
        xdata[name] += [nthreads]
        ydata[name] += [float(row[6])]

import numpy as np
from scipy.optimize import curve_fit as cf

def fixname(s):
    while True:
        n = re.sub(r'(\d)(\d\d\d)\b',r'\1,\2',s)
        if s == n:
            return s
        s = n

symbols = {
    "java" : "D",
    "c++" : "8",
    "cxx" : "8",
    "charm++" : "x",
    "hpx" : "+",
    "julia" : "v",
    "python" : "*",
    "swift" : "^",
    "chapel" : ">",
    "chapelng" : "<",
    "go" : "s",
    "rust" : "p",
    "heat" : "x",
    "heat_ghosts" : "p"
}

colors = {
    "java" : "darkgray",
    "c++" : "dimgray",
    "cxx" : "dimgray",
    "charm++" : "gray",
    "hpx" : "black",
    "julia" : "black",
    "python" : "black",
    "swift" : "dimgray",
    "chapel" : "gray",
    "chapelng" : "darkgray",
    "go" : "silver",
    "rust" : "black",
    "heat" : "black",
    "heat_ghosts" : "gray"
}

lines = {
    "java" : "-",
    "c++" : "--",
    "cxx" : ":",
    "charm++" : "-",
    "hpx" : "--",
    "julia" : ":",
    "python" : "-",
    "swift" : "--",
    "chapel" : ":",
    "chapelng" : "-",
    "go" : "--",
    "rust" : ":",
    "heat" : "-",
    "heat_ghosts" : "--"
}

lineLabels = []
labels = []

for name in xdata:
    xv = np.asarray(xdata[name])
    yv = np.asarray(ydata[name])
    assert len(xv) == len(yv)
    fix = fixname(name)
    symbol = symbols[name]
    color = colors[name]
    labels.append(fix)
    lineLabels.append(Line2D([0], [0], color=color, linewidth=3, linestyle='--', marker=symbol))
    p = plt.semilogy(xv,yv,symbol,color=color)

    def rt(x, k, p, o, o2, o3):
        return k*(1-p + p/x + o*np.log(x) + o2*np.sqrt(x)+o3*x)

    if len(xv) > 3:
        try:
            bounds_upper = [np.inf, 1, 1, 1, 1]
            bounds_lower = [0 for x in bounds_upper]
            bounds = (tuple(bounds_lower), tuple(bounds_upper))
            r  = cf(rt,xv,yv,maxfev=5000,bounds=bounds)
        except Exception as e:
            print("Could not fit curve for:",name,e)
            continue
        k=r[0][0]
        pbar=r[0][1]
        o=r[0][2]
        o2=r[0][3]
        o3=r[0][4]
        print("For ",name,": Parallel=","%.8g" % pbar," using ",len(xv)," run data pts.",sep='')
        overheads = 0
        if o > 1e-14:
            print("   Overhead log(N)=%.8g:" % o),
            overheads += 1
        if o2 > 1e-14:
            print("   Overhead sqrt(N)=%.8g:" % o2),
            overheads += 1
        if o3 > 1e-14:
            print("   Overhead N=%.8g:" % o3),
            overheads += 1
        if overheads == 0:
            print("   No appreciable overheads")
        corr_matrix = numpy.corrcoef(xv,yv)
        corr = corr_matrix[0,1]
        print("r2 = ",corr**2)
        print()
        xv2 = np.asarray(range(1,round(1+max(xdata[name]))))
        yv2 = rt(xv2,*r[0])
        line = lines[name]
        plt.semilogy(xv2,yv2,line,color=color)
        #plt.semilogy(xv2,yv2,'-',label='fit '+fix,color=color)
plt.legend(lineLabels,labels,ncol=4,shadow=True,fancybox=True,loc='upper center', bbox_to_anchor=(0.5, -0.15))
ax = plt.gca()
#ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15),
#          fancybox=True, shadow=True, ncol=4)
plt.grid()
plt.xlabel("\#cores")
plt.ylabel("Time [s]")
plt.title("nx="+nx+" and nt="+nt)
plt.savefig('plot'+file+'.pdf',bbox_inches='tight')
plt.savefig('plot'+file+'.png',bbox_inches='tight')
plt.tight_layout()
print("Saving to plotme.png")
plt.savefig("plotme.png")
plt.show()


from django.db import models
from .product import Product

class Magazine(models.Model):

    product = models.OneToOneField(Product, related_name='Magazines', on_delete=models.CASCADE, primary_key=True)
    name = models.CharField('Name', max_length = 200, blank=True, null=True)
    edition = models.IntegerField(default=0, blank=True, null=True)
    issn = models.CharField('ISSN', max_length = 8, blank=True, null=True)

    PARENT_MODEL= Product
    PARENT_MODEL_NAME = 'product'


from flask import Flask, render_template, redirect, request, jsonify, flash, url_for
from . import student_bp
import app.models.student_model as studModel
from app.student.forms import StudentForm

headings = ("ID_Number", "First Name", "Last Name", "Course", "College", "Year", "Gender", "Actions")

@student_bp.route('/')
def home_page():
    return render_template('home.html')

@student_bp.route('/student')
def student():
    student_info = studModel.Student.all()
    for student in student_info:
        id_number, first_name, last_name, course_code, year_, gender = student

        # Fetch the college codes for the current student's id_number
        college = studModel.Student.get_all_college(id_number)

        if college:
            college_code = college[0]['college_code']
        else:
            college_code = ""  # Set to an empty string if no college code is found

        # Update the student data with the college code
        student_info[student_info.index(student)] = [id_number, first_name, last_name, course_code, college_code, year_, gender]

    return render_template('student.html', headings=headings, data=student_info)

@student_bp.route('/student/add', methods=['POST','GET'])
def add():
    form = StudentForm(request.form)
    course_code = studModel.Student.get_course_code()

    if request.method == 'POST' and form.validate():
        check_id = form.id_number.data
        student_exists = studModel.Student.unique_code(check_id)

        if student_exists:
            flash("ID numder already exists! Please enter a unique ID numbers", 'error')
        else:
            student = studModel.Student(
                id_number=check_id,
                first_name=form.first_name.data,
                last_name=form.last_name.data,
                course_code=form.course_code.data,
                year_=form.year_.data,
                gender=form.gender.data
            )
            student.add()
            flash("Student added successfully!", 'success')
            return redirect(url_for('student.student'))

    return render_template('add_student.html', form=form, course=course_code)

@student_bp.route('/student/edit', methods=["GET", "POST"])
def edit_student():
    id_number = request.args.get('id_number')
    form = StudentForm()
    course_code = studModel.Student.get_course_code()
    student_data = studModel.Student.get_student_by_id(id_number)

    if student_data:
        # Ensure that student_data is not empty before accessing elements
        student_data_dict = {
            "id_number": student_data['id_number'],
            "first_name": student_data['first_name'],
            "last_name": student_data['last_name'],
            "course_code": student_data['course_code'],
            "year_": student_data['year_'],
            "gender": student_data['gender']
        }
    else:
        # Handle the case where no student data was found
        flash("Student not found.", "error")
        return redirect(url_for("student.student"))

    if request.method == "POST" and form.validate():
        new_first_name = form.first_name.data
        new_last_name = form.last_name.data
        new_course_code = form.course_code.data
        new_year = form.year_.data
        new_gender = form.gender.data

        if studModel.Student.update(id_number, new_first_name, new_last_name, new_course_code, new_year, new_gender):
            flash("Student information updated successfully!", "success")
            return redirect(url_for("student.student"))
        else:
            flash("Failed to update student information.", "error")

    return render_template("edit_student.html", form=form, info=student_data_dict, course=course_code)

@student_bp.route("/student/delete", methods=["POST"])
def delete_student():
    try:
        id_number = request.form.get('id_number')
        if studModel.Student.delete(id_number):
            return jsonify(success=True, message="Successfully deleted")
        else:
            return jsonify(success=False, message="Failed")
    except Exception as e:
        # Log the error for debugging purposes
        student_bp.logger.error("An error occurred: %s" % str(e))
        return jsonify(success=False, message="Internal Server Error"), 500

@student_bp.route('/student/search', methods=['POST'])
def search_student():
    try:
        search_query = request.form.get('searchTerm')
        filter_by = request.form.get('filterBy')  # Get the filterBy parameter

        if filter_by == 'all':
            # If filterBy is 'all', perform a general search
            search_results = studModel.Student.search_student(search_query)
        else:
            # Otherwise, filter based on the selected column
            search_results = studModel.Student.filter_student(filter_by, search_query)

        return jsonify(search_results)
    except Exception as e:
        # Handle errors and return an error response
        return jsonify(error=str(e)), 500


import random, uuid

colors = ["red", "orange", "yellow", "blue", "green", "indigo", "violet", "brown", "sienna", "carnation", "citrus", "saffron", "lime", "lavender", "emerald"]
animals = ["cat", "dog", "hamster", "gecko", "ferret", "oppossum", "phoenix", "dinosaur", "spider", "camel", "unicorn", "thestral", "swan"]

def generate_random_url():
    color = colors[random.randint(0, 14)]
    animal = animals[random.randint(0, 12)]
    number = uuid.uuid1()

    return color + "-" + animal + "-" + str(number)


from .defines import set_creds, make_apicall
import json
user_media_model = ""

def get_user_media(params):

    endpoint_params = dict()
    endpoint_params[
        'fields'] = 'timestamp,id,caption,media_type,shortcode,permalink,media_url'
    endpoint_params['access_token'] = params['access_token']

    url = params['endpoint_base'] + \
        params['instagram_account_id'] + '/media'

    response = make_apicall(url, endpoint_params)

    keyed_media_urls = {post['id']: [post['shortcode'], post['permalink'], post['media_url']]
                        for post in response['data']}

    with open("json/" + "keyed_media_urls" + ".json", 'w') as outfile:
        json.dump(keyed_media_urls, outfile)

    return response

def get_comments_and_likes(params):

    endpoint_params = dict()
    endpoint_params[
        'fields'] = 'id,comments,like_count,comments_count'
    endpoint_params['access_token'] = params['access_token']

    url = params['endpoint_base'] + \
        params['instagram_account_id'] + '/media'

    response = make_apicall(url, endpoint_params)

    is_keyed_comments = {}
    is_keyed_photo_metrics = {}

    for obj in response['data']:
        photo_id = obj['id']
        if 'comments' in obj:
            comments = []
            for comment in obj['comments']['data']:
                comments.append(comment['text'])
            is_keyed_comments[photo_id] = comments

    for key in response['data']:
        photo_id = key['id']
        like_count = key['like_count']
        comments_count = key['comments_count']
        count_str = str(like_count) + ' likes, ' + \
            str(comments_count) + ' comments'
        is_keyed_photo_metrics[photo_id] = count_str

    with open("json/" + "is_keyed_comments" + ".json", 'w') as outfile:
        json.dump(is_keyed_comments, outfile)

    with open("json/" + "is_keyed_photo_metrics" + ".json", 'w') as outfile:
        json.dump(is_keyed_photo_metrics, outfile)

    return make_apicall(url, endpoint_params)

def get_long_lived_token(params):

    endpoint_params = dict()
    endpoint_params['grant_type'] = 'fb_exchange_token'
    endpoint_params['client_id'] = params['client_id']
    endpoint_params['client_secret'] = params['client_secret']
    endpoint_params['fb_exchange_token'] = params['short_lived_access_token']

    url = params['endpoint_base'] + 'oauth/access_token'
    response = make_apicall(url, endpoint_params)

    long_lived_token = str(response['access_token'])
    return long_lived_token

def get_page_id(params):

    endpoint_params = dict()
    endpoint_params['access_token'] = params['access_token']

    url = params['endpoint_base'] + 'me/accounts'

    response = make_apicall(url, endpoint_params)

    page_id = str(response['data'][0]['id'])

    return page_id

def get_instagram_account_id(params):

    endpoint_params = dict()
    endpoint_params['access_token'] = params['access_token']
    endpoint_params['fields'] = 'instagram_business_account'

    url = params['endpoint_base'] + params['page_id']

    response = make_apicall(url, endpoint_params)

    instagram_account_id = str(response['instagram_business_account']['id'])

    return instagram_account_id

def get_ig_username(params):

    endpoint_params = dict()
    endpoint_params['access_token'] = params['access_token']
    endpoint_params['fields'] = 'username,name,followers_count,profile_picture_url'

    url = params['endpoint_base'] + params['instagram_account_id']

    response = make_apicall(url, endpoint_params)

    basic_profile_info = [response['username'],
                          response['name'], response['followers_count'], response['profile_picture_url']]

    with open("json/" + "basic_profile_info" + ".json", 'w') as outfile:
        json.dump(basic_profile_info, outfile)

    ig_username = str(response['username'])

    return ig_username

def load_instance(access_token):

    params = get_creds(access_token)
    get_user_media(params)
    get_comments_and_likes(params)

    return print('instance loaded')

def get_creds(access_token):

    params = set_creds()

    params['short_lived_access_token'] = access_token
    params['access_token'] = get_long_lived_token(params)

    params['page_id'] = get_page_id(params)
    params['instagram_account_id'] = get_instagram_account_id(params)
    params['ig_username'] = get_ig_username(params)

    return params

def set_access_token(data):

    params = get_creds()

    return print('new long_lived_token acquired')


import numpy as np
import imageio
import os
from ssr.utility.logging_extension import logger

def read_hdr_image(in_hdr_png):
    # =========================================================================
    # https://github.com/imageio/imageio/issues/204
    # Read with FreeImage instead of Pillow is MANDATORY TO READ 16 bit color
    # pngs correctly!!!!
    hdr_img = imageio.imread(in_hdr_png, "PNG-FI").astype(dtype=np.float64)
    # =========================================================================

    if hdr_img.ndim == 3:
        red_channel = hdr_img[:, :, 0]
        green_channel = hdr_img[:, :, 1]
        blue_channel = hdr_img[:, :, 2]
        assert np.amax(red_channel) > 7
        assert np.amax(green_channel) > 7
        assert np.amax(blue_channel) > 7

    return hdr_img

def write_as_ldr_image(hdr_img, ofp):
    if os.path.exists(ofp):
        os.remove(ofp)
    imageio.imwrite(ofp, hdr_img.astype(dtype=np.uint8))

def write_as_hdr_image(hdr_img, ofp):
    if os.path.exists(ofp):
        os.remove(ofp)
    imageio.imwrite(ofp, hdr_img.astype(dtype=np.uint16))

def split_msi(in_png, r_out_png, g_out_png, b_out_png, target_dtype=np.uint8):
    im = imageio.imread(in_png)

    imageio.imwrite(r_out_png, im[:, :, 0].astype(dtype=target_dtype))
    imageio.imwrite(g_out_png, im[:, :, 1].astype(dtype=target_dtype))
    imageio.imwrite(b_out_png, im[:, :, 2].astype(dtype=target_dtype))

def combine_msi(
    b_out_png, g_out_png, r_out_png, out_png, target_dtype=np.uint8
):
    image_b = imageio.imread(b_out_png)
    image_g = imageio.imread(g_out_png)
    image_r = imageio.imread(r_out_png)

    combined_np = np.dstack(
        (np.array(image_r), np.array(image_g), np.array(image_b))
    )

    imageio.imwrite(out_png, combined_np.astype(dtype=target_dtype))

def tone_map_channel(channel, percentile=0.5, exp=1.0 / 2.2):

    # =========================================================================
    # Use QGIS to examine the PNG result (i.e. color range of the created file)
    # =========================================================================

    # gamma correction
    # Could be also computed with GDAL
    # https://gdal.org/programs/gdal_translate.html#cmdoption-gdal-translate-exponent
    tone_mapped_channel = np.power(channel, exp)

    # cut off the small values
    below_thres = np.percentile(
        tone_mapped_channel.reshape((-1, 1)), percentile
    )
    tone_mapped_channel[tone_mapped_channel < below_thres] = below_thres
    # cut off the big values
    above_thres = np.percentile(
        tone_mapped_channel.reshape((-1, 1)), 100 - percentile
    )
    tone_mapped_channel[tone_mapped_channel > above_thres] = above_thres

    # Put in range 0 - 255
    tone_mapped_channel = (
        255 * (tone_mapped_channel - below_thres) / (above_thres - below_thres)
    )
    return tone_mapped_channel

def tone_map_separate(im, percentile=0.5, exp=1.0 / 2.2):
    red_channel = im[:, :, 0]
    green_channel = im[:, :, 1]
    blue_channel = im[:, :, 2]

    red_channel = tone_map_channel(red_channel, percentile, exp)
    green_channel = tone_map_channel(green_channel, percentile, exp)
    blue_channel = tone_map_channel(blue_channel, percentile, exp)

    hdr_result_tone_mapped = np.dstack(
        (
            np.array(red_channel),
            np.array(green_channel),
            np.array(blue_channel),
        )
    )

    return hdr_result_tone_mapped

def rgb_2_gray(rgb):
    weights = get_rgb_2_gray_weights()
    return (
        weights[0] * rgb[..., 0]
        + weights[1] * rgb[..., 1]
        + weights[2] * rgb[..., 2]
    )

def get_rgb_2_gray_weights(use_luminance_weights=True):
    # https://brohrer.github.io/convert_rgb_to_grayscale.html
    #   Channel-dependent luminance perception
    #       Y_linear = 0.2126 R + 0.7152 G + 0.0722 B
    #   Gamma compression
    #       ...
    #   Linear approximation
    #       Y' = 0.299 + 0.587 G + 0.114 B
    #   Which one should I use?
    #      If close is good enough or if you really care about speed, use the
    #      linear approximation of gamma correction.
    #      This is the approach used by MATLAB, Pillow, and OpenCV

    # https://stackoverflow.com/questions/12201577/how-can-i-convert-an-rgb-image-into-grayscale-in-python
    #   gray = 0.2989 * r + 0.5870 * g + 0.1140 * b
    if use_luminance_weights:
        return [0.2989, 0.5870, 0.1140]
    else:
        return [1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0]

def tone_map_joint(im, percentile=0.5, exp=1.0 / 2.2):

    # https://en.wikipedia.org/wiki/Gamma_correction
    # gamma correction
    im = np.power(im, exp)

    im_gray = rgb_2_gray(im)

    im_gray_1d = im_gray.reshape((-1, 1))

    # cut off the small values
    below_thres = np.percentile(im_gray_1d, percentile)
    below_thresh_indices = im_gray < below_thres
    below_thres_arr = np.full_like(im_gray, below_thres)

    below_scale_factor_arr = np.divide(below_thres_arr, im_gray)
    r_below_scaled = np.multiply(im[:, :, 0], below_scale_factor_arr)
    g_below_scaled = np.multiply(im[:, :, 1], below_scale_factor_arr)
    b_below_scaled = np.multiply(im[:, :, 2], below_scale_factor_arr)

    im[below_thresh_indices, 0] = r_below_scaled[below_thresh_indices]
    im[below_thresh_indices, 1] = g_below_scaled[below_thresh_indices]
    im[below_thresh_indices, 2] = b_below_scaled[below_thresh_indices]

    # cut off the big values
    above_thres = np.percentile(im_gray_1d, 100 - percentile)
    above_thresh_indices = im_gray > above_thres
    above_thres_arr = np.full_like(im_gray, above_thres)

    above_scale_factor_arr = np.divide(above_thres_arr, im_gray)
    r_above_scaled = np.multiply(im[:, :, 0], above_scale_factor_arr)
    g_above_scaled = np.multiply(im[:, :, 1], above_scale_factor_arr)
    b_above_scaled = np.multiply(im[:, :, 2], above_scale_factor_arr)

    im[above_thresh_indices, 0] = r_above_scaled[above_thresh_indices]
    im[above_thresh_indices, 1] = g_above_scaled[above_thresh_indices]
    im[above_thresh_indices, 2] = b_above_scaled[above_thresh_indices]

    # === Verify results ===
    # logger.info(f'below_thres {below_thres}')
    # logger.info(f'above_thres {above_thres}')
    # im_gray_ref = rgb_2_gray(im)
    # im_gray_ref_1d = im_gray_ref.reshape((-1, 1))
    # below_thres_ref = np.percentile(im_gray_ref_1d, percentile)
    # above_thres_ref = np.percentile(im_gray_ref_1d, 100 - percentile)
    # logger.info(f'below_thres_ref {below_thres_ref}')
    # logger.info(f'above_thres_ref {above_thres_ref}')
    # assert np.allclose(below_thres, below_thres_ref)
    # assert np.allclose(above_thres, above_thres_ref)

    # log_value(im, 0, print_min=True)
    # log_value(im, 1, print_min=True)
    # log_value(im, 2, print_min=True)
    #
    # log_value(im, 0, print_min=False)
    # log_value(im, 1, print_min=False)
    # log_value(im, 2, print_min=False)

    # Put in range 0 - 255
    # logger.info(
    #   f'min values: {np.amin(im)}, {np.amin(im[:, :, 0])}, {np.amin(im[:, :, 1])}, {np.amin(im[:, :, 2])}'
    # )
    # logger.info(
    #   f'max values: {np.amax(im)}, {np.amax(im[:, :, 0])}, {np.amax(im[:, :, 1])}, {np.amax(im[:, :, 2])}'
    # )
    # === ===

    # im = rescale_im_or_chan(im)
    im[:, :, 0] = rescale_im_or_chan(im[:, :, 0])
    im[:, :, 1] = rescale_im_or_chan(im[:, :, 1])
    im[:, :, 2] = rescale_im_or_chan(im[:, :, 2])
    return im

def rescale_im_or_chan(im_or_chan):
    min_val = np.amin(im_or_chan)
    max_val = np.amax(im_or_chan)
    im_or_chan = 255 * (im_or_chan - min_val) / (max_val - min_val)
    return im_or_chan

def log_value(im, chan_idx, print_min):
    logger.info("------------------")
    logger.info(f"chan_idx {chan_idx}")
    logger.info(f"print_min {print_min}")
    chan = im[:, :, chan_idx]
    if print_min:
        val = chan.argmin()
    else:
        val = chan.argmax()

    idx_2d = np.unravel_index(val, chan.shape)
    weights = get_rgb_2_gray_weights()
    rgb = im[idx_2d]
    gray_ref = np.dot(weights, rgb)

    logger.info(f"idx_2d {idx_2d}")
    logger.info(f"chan value {chan[idx_2d]}")
    logger.info(f"rgb value {rgb}")
    logger.info(f"gray_ref {gray_ref}")
    logger.info("------------------")

# hdr_img is 16-bit, while ldr_img is 8 bit
def tone_map_hdr_to_ldr_general(
    hdr_img_ifp,
    ldr_img_ofp,
    joint_tone_mapping=True,
    percentile=0.5,
    exp=1.0 / 2.2,
):

    im = read_hdr_image(hdr_img_ifp)

    # A single (PAN chromatic / grey) channel has 2 spatial dimension
    if im.ndim == 2:
        hdr_result_tone_mapped = tone_map_channel(im, percentile, exp)

    # A MSI / color image has 3 channels
    elif im.ndim == 3:
        if joint_tone_mapping:
            hdr_result_tone_mapped = tone_map_joint(im, percentile, exp)
        else:
            hdr_result_tone_mapped = tone_map_separate(im, percentile, exp)
    else:
        assert False

    write_as_ldr_image(hdr_result_tone_mapped, ldr_img_ofp)


#!/usr/bin/env python
# coding: utf-8

import sys

import pandas as pd
import seaborn as sns

import matplotlib.pyplot as plt

def main():

    filenames = sys.argv[1:]

    df = pd.concat(pd.read_csv(filename) for filename in filenames)

    names = df['name'].unique()

    for name in names:
        plt.clf()
        custom = df[df["name"]==name]
        custom = custom[custom["size"] >= 20]
        plt.title(name)
        chart = sns.pointplot(data=custom, x="size", y="us", hue="domain")
        chart.set_xticklabels(chart.get_xticklabels(), rotation=45)
        plt.savefig(f"{name}.png")

if __name__ == '__main__':
    main()


import telebot
from telebot import types
import requests
import time
import os
import json

vk_token = 'Your vk token'
tg_token ='Your tg token'
id_channel = '@your id channel'
group_name = 'your group name'
URL = 'your group url'

with open('old_id.json') as file:
    old_id = json.load(file)

bot = telebot.TeleBot(tg_token)
print('bot is online')
while True:
    url = f'https://api.vk.com/method/wall.get?domain={group_name}&count=10&access_token={vk_token}&v=5.81'
    req = requests.get(url)
    src = req.json()
    posts = src['response']['items']
    new_post = posts[1]

    if new_post["id"] != old_id['id']:
        print('----New Post!----')
        LINK = f'{URL}{new_post["from_id"]}_{new_post["id"]}'

        markup = types.InlineKeyboardMarkup()
        button1 = types.InlineKeyboardButton("Перейти к записи 🛫", url=LINK)
        markup.add(button1)
        try:
            photo_url =new_post['attachments'][0]['photo']['sizes'][3]['url']
            img = requests.get(photo_url)
            with open(f'{new_post["id"]}.jpg', 'wb') as img_file:
                img_file.write(img.content)

            print(f'photo_{new_post["id"]} is loaded!   ||  {LINK}')
            pic = open(f'{new_post["id"]}.jpg','rb')
            text = f'{new_post["text"]}'
            bot.send_photo(id_channel, pic, f'{text[:278]}...Читать полностью', parse_mode='html', reply_markup=markup)
            pic.close()
            os.remove(f'{new_post["id"]}.jpg')
            print('photo deleted')
        except Exception:
            print(f'No photo or any something Exceptions...   ||  {LINK}')
            bot.send_message(id_channel, f'Новый пост в нашем сообществе. Бегом смотреть!\n\n{LINK}',reply_markup=markup)

        old_id = new_post
        with open('old_id.json', 'w') as outfile:
            json.dump(old_id, outfile)

        print('-'*30)

    time.sleep(3600)

bot.infinity_polling()


import os

from setuptools import setup

HERE = os.path.abspath(os.path.dirname(__file__))

with open(os.path.join(HERE, 'README.md')) as f:
    README = f.read()

setup(
    name='pytest-dynamic-fixtures',
    long_description=README,
    version='0.0.1',
    install_requires=["pytest"],
    packages=["pytest_dynamic_fixtures"],
    author='Leon Patmore',
    description=''
)


import os,sys
import numpy as np
import matplotlib.pyplot as plt
import cv2
from tqdm import tqdm
from ulitities.base_functions import get_file

inputdir = '/home/omnisky/PycharmProjects/data/tree/isprs/label_all'
outputdir = '/home/omnisky/PycharmProjects/data/tree/isprs/label_binary'
target_class=[1,2,3,4,5]

if __name__=='__main__':
    if not os.path.isdir(inputdir):
        print("Error: input directory is not existed")
        sys.exit(-1)
    if not os.path.isdir(outputdir):
        print("Warning: output directory is not existed")
        os.mkdir(outputdir)

    for t in tqdm(target_class):
        files,_ = get_file(inputdir)

        s_dir = os.path.join(outputdir,('t_'+str(t)))
        if not os.path.exists(s_dir):
            os.mkdir(s_dir)

        for file in files:
            label = cv2.imread(file, cv2.IMREAD_GRAYSCALE)
            label = np.array(label,np.uint8)
            # plt.figure("s")
            # plt.imshow(label,cmap="flag")
            # plt.show()
            if not t in np.unique(label):
                print("no label {} in {}".format(t, file))
                continue
            else:
                out_file = os.path.join(s_dir,os.path.split(file)[1])
                data = np.zeros(label.shape,np.uint8)
                index = np.where(label==t)
                data[index]=1
                # plt.figure(2)
                # plt.imshow(data)
                # plt.show()
                # print("sdf")
                cv2.imwrite(out_file,data)


import matplotlib.pyplot as plt
import numpy as np
from scipy.interpolate import griddata
from fast_histogram import histogram2d
x = np.random.normal(0, 1, 10000)
y = np.random.normal(0, 1, 10000)

X = np.linspace(-2, 2, 10000)
Y = np.linspace(-2, 2, 10000)
X, Y = np.meshgrid(X, Y)

def convert_bin_edges_to_points(arr):
    return np.array([(i + j) / 2.0 for i, j in zip(arr[:-1], arr[1:])])

def convert_meshgrid_to_arrays(X, Y, Z):
    """Converts (n,m) arrays to array of (n*m) enteries"""
    return X.flatten(), Y.flatten(), Z.flatten()

# if density:
#     # calculate the probability density function
#     s = hist.sum()
#     for i in _range(D):
#         shape = np.ones(D, int)
#         shape[i] = nbin[i] - 2
#         hist = hist / dedges[i].reshape(shape)
#     hist /= s

H, xedges, yedges = histogram2d(x, y, normed=True)
xpos, ypos = np.meshgrid(convert_bin_edges_to_points(xedges),
                         convert_bin_edges_to_points(yedges))

fig, ax = plt.subplots(nrows=2, ncols=2)
# Plot the model function and the randomly selected sample points
ax[0, 0].contourf(xpos, ypos, H)
ax[0, 0].scatter(x, y, alpha=0.2, marker='.')
ax[0, 0].set_title('Sample points on f(X,Y)')

# Interpolate using three different methods and plot_posterior_predictive_check
# for i, method in enumerate(('nearest', 'linear', 'cubic')):
#     train_x, train_y, train_z = convert_meshgrid_to_arrays(xpos, ypos, H)
#     Ti = griddata((train_x, train_y), train_z, (X, Y), method=method)
#     r, c = (i + 1) // 2, (i + 1) % 2
#     ax[r, c].contourf(X, Y, Ti)
#     ax[r, c].set_title("method = '{}'".format(method))

plt.tight_layout()
plt.show()


import sys
import logging
import gzip
from argparse import ArgumentParser
from typing import Dict, Iterator, List, Optional, Tuple

from dae.utils.verbosity_configuration import VerbosityConfiguration
from dae.genomic_resources import build_genomic_resource_repository
from dae.genomic_resources.genomic_context import GenomicContext
from dae.genomic_resources.genomic_context import get_genomic_context
from dae.genomic_resources.repository import GenomicResourceRepo
from dae.genomic_resources.gene_models import GeneModels, \
    build_gene_models_from_file, \
    build_gene_models_from_resource
from dae.genomic_resources.reference_genome import ReferenceGenome
from dae.genomic_resources.reference_genome import \
    build_reference_genome_from_file
from dae.genomic_resources.reference_genome import \
    build_reference_genome_from_resource

from dae.effect_annotation.annotator import EffectAnnotator
from dae.effect_annotation.effect import AnnotationEffect

logger = logging.getLogger(__name__)

class EffectAnnotatorBuilder:
    @staticmethod
    def set_arguments(parser: ArgumentParser) -> None:
        genomic_repository_group = parser.add_argument_group(
            "Genomic repository options")

        genomic_repository_group.add_argument(
            "-grr", "--genomic-repository-config-filename",
            help="Genomic Repository Configuraiton File"
        )

        gene_model_file_group = parser.add_argument_group(
            "Gene models file options")

        gene_model_file_group.add_argument(
            "-gmfn", "--gene-models-filename",
            help="Gene models file name."
        )
        gene_model_file_group.add_argument(
            "-gmff", "--gene-models-fileformat",
            help="Gene models file format."
        )
        gene_model_file_group.add_argument(
            "-gmmfn", "--gene-mapping-filename",
            help="Gene mapping file.",
            default=None,
            action="store",
        )
        gene_model_resource_group = parser.add_argument_group(
            "Gene models resource options")
        gene_model_resource_group.add_argument(
            "-gmri", "--gene-models-resource-id",
            help="Gene models resource id."
        )

        reference_genome_file_group = parser.add_argument_group(
            "Reference genome file options")
        reference_genome_file_group.add_argument(
            "-rgfn", "--reference-genome-filename",
            help="Reference genome file name."
        )

        reference_genome_resource_group = parser.add_argument_group(
            "Reference genome resource options")
        reference_genome_resource_group.add_argument(
            "-rgri", "--reference-genome-resource-id",
            help="Reference genome resource id."
        )

        parser.add_argument(
            "-pl", "--promoter-len",
            help="promoter length",
            default=0,
            type=int,
        )

    def __init__(self, args):
        self.args = args
        self._gc: Optional[GenomicContext] = None

    def get_genomic_context(self) -> GenomicContext:
        if not self._gc:
            self._gc = get_genomic_context()
        return self._gc

    def get_grr(self) -> GenomicResourceRepo:
        grr: Optional[GenomicResourceRepo] = None

        if self.args.genomic_repository_config_filename:
            grr = build_genomic_resource_repository(
                self.args.genomic_repository_config_filename)
        else:
            grr = self.get_genomic_context().get_genomic_resources_repository()
        if grr:
            return grr
        return build_genomic_resource_repository()

    def get_gene_models(self) -> GeneModels:
        if self.args.gene_models_resource_id:
            resource = self.get_grr().get_resource(
                self.args.gene_models_resource_id)
            return build_gene_models_from_resource(resource).load()

        if self.args.gene_models_filename:
            return build_gene_models_from_file(
                self.args.gene_models_filename,
                self.args.gene_models_fileformat,
                self.args.gene_mapping_filename)

        gm = self.get_genomic_context().get_gene_models()
        if gm:
            return gm
        raise ValueError("No gene models could be found!")

    def get_refernce_genome(self) -> ReferenceGenome:
        if self.args.reference_genome_resource_id:
            resource = self.get_grr().get_resource(
                self.args.reference_genome_resource_id)
            return build_reference_genome_from_resource(resource)

        if self.args.reference_genome_filename:
            return build_reference_genome_from_file(
                self.args.reference_genome_filename)

        ref = self.get_genomic_context().get_reference_genome()
        if ref:
            return ref
        raise ValueError("No reference genome could be found!")

    def build_effect_annotator(self):
        ref = self.get_refernce_genome()
        gm = self.get_gene_models()
        logger.info(
            "Building effect annotator with:\n"
            "\treferende genome from %s,\n"
            "\tgene models from %s, and\n"
            "\tpromoter lengths of %s",
            ref.resource.resource_id, gm.resource.resource_id,
            self.args.promoter_len)
        return EffectAnnotator(ref, gm, promoter_len=self.args.promoter_len)

class VariantColumnInputFile:
    variant_columns = ["chrom", "pos",
                       "location", "variant", "ref", "alt"]

    @staticmethod
    def set_argument(parser: ArgumentParser):
        variant_columns_group = parser.add_argument_group(
            "Variant columns")
        for variant_column in VariantColumnInputFile.variant_columns:
            variant_columns_group.add_argument(
                f"--{variant_column}-col",
                default=variant_column)
        input_file_group = parser.add_argument_group(
            "Input file")
        input_file_group.add_argument("input_filename",
                                      nargs="?", default="-")
        input_file_group.add_argument("--input-sep", default="\t")

    def __init__(self, args):
        self.args = args

        # open file
        filename = self.args.input_filename
        if filename == "-":
            self.file = sys.stdin
        else:
            if filename.endswith("gz"):
                self.file = gzip.open(filename, "rt")
            else:
                self.file = open(filename, "rt")  # pylint: disable=R1732

        # read header
        self.header: List[str] = self.file.readline(). \
            strip("\n\r").split(self.args.input_sep)

        # prepare the variant_attribute_index
        args_dict = vars(self.args)
        self.variant_attribute_index: Dict[str, int] = {}
        for variant_attribute in VariantColumnInputFile.variant_columns:
            col = args_dict[f"{variant_attribute}_col"]
            idx = [i for i, h in enumerate(self.header) if h == col]
            if len(idx) > 1:
                raise ValueError(
                    f"the column {col} is repeated twice in the "
                    f"input file {filename}")
            if len(idx) == 1:
                self.variant_attribute_index[variant_attribute] = idx[0]

    def get_header(self) -> List[str]:
        return self.header

    def get_lines(self) -> Iterator[Tuple[List[str], Dict[str, str]]]:
        for line in self.file:
            columns = line.strip("\n\r").split(self.args.input_sep)
            variant_attributes = {
                k: columns[i] for k, i in self.variant_attribute_index.items()
            }
            yield columns, variant_attributes

    def close(self) -> None:
        self.file.close()

class VariantColumnOutputFile:
    @staticmethod
    def set_argument(parser: ArgumentParser):
        output_file_group = parser.add_argument_group(
            "Output file")
        output_file_group.add_argument("output_filename",
                                       nargs="?", default="-")
        output_file_group.add_argument("--output-sep", default="\t")

    def __init__(self, args):
        self.args = args

        filename = self.args.output_filename
        if filename == "-":
            self.file = sys.stdout
        else:
            if filename.endswith("gz"):
                self.file = gzip.open(filename, "wt")
            else:
                self.file = open(filename, "wt")  # pylint: disable=R1732

    def write_columns(self, columns) -> None:
        print(*columns, sep=self.args.output_sep, file=self.file)

    def close(self) -> None:
        self.file.close()

class AnnotationAttributes:
    ALL_ATTRIBUTE_IDXS = {aa: aai for aai, aa in enumerate(
        ["worst_effect", "gene_effects", "effect_details"])}

    @staticmethod
    def set_argument(parser: ArgumentParser, default_columns: str =
                     "worst_effect,gene_effects,effect_details"):
        annotation_attributes_group = parser.add_argument_group(
            "Annotation attributes.")
        annotation_attributes_group. \
            add_argument("--annotation-attributes",
                         default=default_columns,
                         help="The available attributes are "
                         "worst_effect, gene_effects, and effect_details. "
                         "Examples: "
                         "1. 'worst_effect, gene_effects';"
                         "2. 'WE: worst_effect, GE: gene_effects, "
                         "ED: effect_details'"
                         f"The default is : '{default_columns}'.")

    def __init__(self, args):
        self.args = args
        self.out_attributes: List[str] = []
        self.source_attributes: List[str] = []

        for attribute_def in self.args.annotation_attributes.split(","):
            parts = attribute_def.split(":")
            if len(parts) == 1:
                out = parts[0]
                source = parts[0]
            elif len(parts) == 2:
                out = parts[0]
                source = parts[1]
            else:
                raise ValueError(
                    "Incorrect attribute definition. "
                    "TO IMPROVE MESSAGE!")
            self.out_attributes.append(out)
            self.source_attributes.append(source)
        if len(self.out_attributes) != len(set(self.out_attributes)):
            raise ValueError(
                "The output attributes should not be repeated. "
                "TO IMPROVE MESSAGE!!!")
        self.value_idxs = [AnnotationAttributes.ALL_ATTRIBUTE_IDXS[aa]
                           for aa in self.source_attributes]

    def get_out_attributes(self) -> List[str]:
        return self.out_attributes

    def get_source_attributes(self) -> List[str]:
        return self.source_attributes

    def get_values(self, effect) -> List[str]:
        full_desc = AnnotationEffect.effects_description(effect)
        return [full_desc[idx] for idx in self.value_idxs]

def cli_columns():
    parser = ArgumentParser(
        description="Annotate Variant Effects in a Column File.")

    VerbosityConfiguration.set_argumnets(parser)
    EffectAnnotatorBuilder.set_arguments(parser)
    VariantColumnInputFile.set_argument(parser)
    VariantColumnOutputFile.set_argument(parser)
    AnnotationAttributes.set_argument(parser)

    args = parser.parse_args(sys.argv[1:])
    VerbosityConfiguration.set(args)
    annotator = EffectAnnotatorBuilder(args).build_effect_annotator()
    input_file = VariantColumnInputFile(args)
    output_file = VariantColumnOutputFile(args)
    annotation_attributes = AnnotationAttributes(args)

    output_file.write_columns(
        input_file.get_header() + annotation_attributes.get_out_attributes())
    for columns, vdef in input_file.get_lines():
        logger.debug("vdef: %s AAA", vdef)
        effect = annotator.do_annotate_variant(**vdef)
        output_file.write_columns(
            columns + annotation_attributes.get_values(effect)
        )

def cli_vcf():
    # pylint: disable=C0415
    # type: ignore
    import pysam

    parser = ArgumentParser(
        description="Annotate Variant Effects in a VCF file.")

    VerbosityConfiguration.set_argumnets(parser)
    EffectAnnotatorBuilder.set_arguments(parser)
    AnnotationAttributes.set_argument(
        parser,
        default_columns="WE:worst_effect,GE:gene_effects,ED:effect_details")

    parser.add_argument("input_filename", help="input VCF variants file name")
    parser.add_argument(
        "output_filename", nargs="?", help="output file name (default: stdout)"
    )
    args = parser.parse_args(sys.argv[1:])
    VerbosityConfiguration.set(args)
    annotator = EffectAnnotatorBuilder(args).build_effect_annotator()
    annotation_attributes = AnnotationAttributes(args)

    # pylint: disable=no-member
    infile = pysam.VariantFile(args.input_filename)
    if args.output_filename is None:
        outfile = sys.stdout
    else:
        outfile = open(args.output_filename, "w")  # pylint: disable=R1732

    # handling the header
    header = infile.header
    header.add_meta(
        "variant_effect_annotation", "GPF variant effects annotation."
    )
    header.add_meta(
        "variant_effect_annotation_command", f'"{" ".join(sys.argv)}"'
    )
    source_attribute_description = {
        "worst_effect": "The worst effect.",
        "gene_effects": "Gene effects.",
        "effect_details": "Detailed effects."
    }
    for out_a, src_a in zip(annotation_attributes.get_out_attributes(),
                            annotation_attributes.get_source_attributes()):
        header.info.add(out_a, "A", "String",
                        source_attribute_description[src_a])

    print(str(header), file=outfile, end="")

    # handling the variants
    out_as = annotation_attributes.get_out_attributes()
    for variant in infile:
        buffers = [list([]) for _ in out_as]

        for alt in variant.alts:
            effect = annotator.do_annotate_variant(
                chrom=variant.chrom,
                pos=variant.pos,
                ref=variant.ref,
                alt=alt
            )
            for buff, v in zip(buffers,
                               annotation_attributes.get_values(effect)):
                buff.append(v)

        for out_a, buff in zip(out_as, buffers):
            variant.info[out_a] = ",".join(buff)
        print(str(variant), file=outfile, end="")

    infile.close()
    if args.output_filename:
        outfile.close()

if __name__ == "__main__":
    cli_columns()


from src.utils.File_reader import Reader
from src.Gene import *
from src.Individual import Individual
from src.Environment import Group, Env
from numpy import random
import logging

logging.basicConfig(filename='./syslogging.log',
                    format='[%(asctime)s-%(filename)s-%(levelname)s:%(message)s]',
                    filemode='a',
                    level=logging.DEBUG,
                    datefmt='%Y-%m-%d%I:%M:%S %p')

FILE_PUCKS = './data/Pucks.csv'
FILE_GATES = './data/Gates.csv'
FILE_TICKETS = './data/Tickets.csv'

INIT_POPULATION = 5000
LIMIT = 20000
FILTER_RATE = 0.4
MUTATION_RATE1 = 0.01
MUTATION_RATE2 = 0.05
MUTATION_RATE3 = 0.15
ENV_MUTATE_RATE = 0.4
ALPHA = 10
BETA = 1

EXIT = 30000

class Main:

    def __init__(self, filter_rate, mutation_rate1, mutation_rate2, mutation_rate3, env_mutate_rate):
        self._filter_rate = filter_rate
        self._mutation_rate1 = mutation_rate1
        self._mutation_rate2 = mutation_rate2
        self._mutation_rate3 = mutation_rate3
        self._env_mutate_rate = env_mutate_rate

    @staticmethod
    def get_object():
        df_bucks = Reader.read(FILE_PUCKS)
        flights = Reader.form_flights(df_bucks)
        df_gates = Reader.read(FILE_GATES)
        gates = Reader.form_gates(df_gates)
        df_tickets = Reader.read(FILE_TICKETS)
        tickets = Reader.form_ticket(df_tickets)
        Individual.set_tickets(tickets)
        return flights, gates, tickets

    @staticmethod
    def form_individual():
        flights, gates, tickets = Main.get_object()
        gene_list = []
        for f in flights:
            while True:
                random_index = random.randint(0, len(gates)-1)
                random_gate = gates[random_index]
                if f._type_arr not in random_gate._type_arr or \
                        f._type_dep not in  random_gate._type_dep or \
                        f._type_aircraft not in random_gate._type_flight:
                    continue
                else:
                    gene_list.append(Gene(f, random_gate))
                    break
        genotype = GenoType(gene_list)
        if genotype.validation() is False:
            return None
        else:
            return Individual(genotype)

    @staticmethod
    def form_pop():
        logging.debug("Create init population")
        ind_list = []
        while len(ind_list) < INIT_POPULATION:
            tempt = Main.form_individual()
            if tempt is None:
                continue
            else:
                ind_list.append(tempt)
        return Group(ind_list, LIMIT)

    def run(self):
        env = Env(group=Main.form_pop(),
                  filter_rate=self._filter_rate,
                  mutation_rate=self._mutation_rate3,
                  gates=Main.get_object()[1],
                  env_mutate_rate=self._env_mutate_rate)
        cur_mutation_rate = 3
        n = 0
        while True:
            logging.debug('=============================================================================')
            logging.debug('This is the {}th generation.'.format(n))

            env.next_generation()
            env.sort()
            pop_size = env.get_population_size()
            median_score = env.get_ind_by_index(int(pop_size/2)).get_pheno()
            highest_score = env.get_ind_by_index(-1).get_pheno()
            env.filter()

            logging.debug('Highest score is {}'.format(highest_score))
            logging.debug('Median score is {}'.format(median_score))
            if median_score > 0.01 and cur_mutation_rate == 3:
                logging.debug('Changing mutation rate to 0.05')
                env._mutation_rate = self._mutation_rate2
                cur_mutation_rate = 2
            if highest_score > 0.1 and cur_mutation_rate == 2:
                logging.debug('Changing mutation rate to 0.01')
                env._mutation_rate = self._mutation_rate1
                cur_mutation_rate = 1
            if n == EXIT:
                break
            if n%1000 == 0:
                file = open('./result/optimal{}.txt'.format(n), 'w+')
                file.write(env.get_ind_by_index(-1).tostring())
                file.close()
            n += 1

if __name__ == '__main__':
    Individual.set_al_be(ALPHA, BETA)
    main_func = Main(FILTER_RATE, MUTATION_RATE1, MUTATION_RATE2, MUTATION_RATE3, ENV_MUTATE_RATE)
    main_func.run()


import gzip
import argparse
import numpy as np

TrainImagePath = 'C:/Users/jackson/Desktop/chiacheng/visualstudiocode/nctu110fall/ML/hw2/train-images-idx3-ubyte.gz'
TrainLablePath = 'C:/Users/jackson/Desktop/chiacheng/visualstudiocode/nctu110fall/ML/hw2/train-labels-idx1-ubyte.gz'
TestImagePath = 'C:/Users/jackson/Desktop/chiacheng/visualstudiocode/nctu110fall/ML/hw2/t10k-images-idx3-ubyte.gz'
TestLablePath = 'C:/Users/jackson/Desktop/chiacheng/visualstudiocode/nctu110fall/ML/hw2/t10k-labels-idx1-ubyte.gz'
np.set_printoptions(threshold=np.inf)

def readImage(path):
    with gzip.open(path, 'r') as f:
        # first 16 bytes: 4 for magic number, 4 for number of Images, 4 for number of rows, and 4 for number of columns
        mNumber = int.from_bytes(bytes=f.read(4), byteorder='big', signed=False)
        nImage = int.from_bytes(bytes=f.read(4), byteorder='big', signed=False)
        nrow = int.from_bytes(bytes=f.read(4), byteorder='big', signed=False)
        ncol = int.from_bytes(bytes=f.read(4), byteorder='big', signed=False)

        # other bytes: pixels of Image (each pixel is an unsigned byte)
        Images = np.frombuffer(f.read(), dtype=np.uint8).reshape(nImage, nrow, ncol)
        Xdata = Image(mNumber, nImage, nrow, ncol, Images)
    return Xdata

def readLable(path):
    with gzip.open(path, 'r') as f:
        # first 8 bytes: 4 for magic number, 4 for number of Lables
        mNumber = int.from_bytes(bytes=f.read(4), byteorder='big', signed=False)
        nLable = int.from_bytes(bytes=f.read(4), byteorder='big', signed=False)

        # other bytes: lables (each lable is an unsigned byte)
        Lables = np.frombuffer(f.read(), dtype=np.uint8)
        Ydata = Lable(mNumber, nLable, Lables)
    return Ydata

class Image:
    def __init__(self, mNumber, nImage, nrow, ncol, Images):
        self.MagicNumber = mNumber
        self.nImage = nImage
        self.nrow = nrow
        self.ncol = ncol
        self.Images = Images

class Lable:
    def __init__(self, mNumber, nLable, Lables):
        self.MagicNumber = mNumber
        self.nLable = nLable
        self.Lables = Lables

class DiscreteNBClassifer:
    """Initialize a Naive Bayes Classifier for Discrete Mode

    Variables:
        pfrequency: The number of occurence of (label, pixel, bin) = (i, j, k)
        pprobability: The probability of occurence of (label, pixel, bin) = (i, j, k)
        lfrequency: The number of occurence of label i
        lcount: The total number of labels
        lprobability: The probability of occurence of label i
    """
    def __init__(self):
        # 10 for labels: 0 ~ 9
        # 784 for features: 28x28 pixels
        # 32 for bins: make "gray level 256" to 32 bins. That is, each bin has 8 gray levels
        self.pfrequecy = np.zeros(shape=(10, 784, 32), dtype=int)
        self.pprobability = np.zeros(shape=(10, 784, 32), dtype=float)
        self.lfrequency = np.zeros(shape=(10), dtype=int)
        self.lcount = 0
        self.lprobability = np.zeros(shape=(10), dtype=float)

    def train(self, Xtrain: Image, Ytrain: Lable):
        nImage, nPixel = Xtrain.nImage, (Xtrain.nrow * Xtrain.ncol)
        self.lcount = Ytrain.nLable
        # Tally the values of each pixel into 32 bins
        bins = np.floor(Xtrain.Images / 8).reshape(nImage, nPixel).astype(int)

        # Update the frequency, count for pixel and label
        for idxImage in range(nImage):
            curlable = Ytrain.Lables[idxImage]
            self.lfrequency[curlable] += 1

            for idxPixel in range(nPixel):
                self.pfrequecy[curlable][idxPixel][bins[idxImage][idxPixel]] += 1

        # Calculate the probability of each lable
        self.lprobability = self.lfrequency / self.lcount

        # Laplace Smoothing
        self.pfrequecy += 1

        for idxlabel in range(10):
            # Calculate the probability of occurence of (label, pixel, bin) = (i, j, k)
            self.pprobability[idxlabel]= self.pfrequecy[idxlabel]/ (self.lfrequency[idxlabel]+32)

    def predict(self, Xtest: Image):
        nImage, nPixel = Xtest.nImage, (Xtest.nrow * Xtest.ncol)
        predictY = np.zeros(shape=(nImage), dtype=int)
        posterior = np.zeros(shape=(nImage, 10), dtype=float)
        bins = np.floor(Xtest.Images / 8).reshape(nImage, nPixel).astype(int)
        idx = np.arange(nPixel)

        for idxImage in range(nImage):
            for idxLable in range(10):
                # calculate likelohood, prior and posterior in log scale
                likelihood = np.sum(np.log(self.pprobability[idxLable, idx, bins[idxImage][idx]]))
                prior = np.log(self.lprobability[idxLable])
                posterior[idxImage][idxLable] = likelihood + prior

            predictY[idxImage] = np.argmax(posterior[idxImage])
            # marginalize posterior to let them sum up to 1
            posterior[idxImage] /= np.sum(posterior[idxImage])

        return predictY, posterior

    def calError(self, Ypredict, Ytest: Lable):
        nLable = Ytest.nLable
        wrong = np.sum(Ypredict != Ytest.Lables)
        return wrong / nLable

    def Imagination(self):
        imagination = np.zeros(shape=(10, 784), dtype=np.uint8)
        for idxLable in range(10):
            for idxPixel in range(784):
                imagination[idxLable][idxPixel] = np.argmax([np.sum(self.pfrequecy[idxLable][idxPixel][:16]), np.sum(self.pfrequecy[idxLable][idxPixel][16:])])

        print("Imagination of numbers in Bayesian classifier:\n")
        for idxLabel in range(10):
            print(idxLabel, ":")
            print(imagination[idxLabel].reshape(28, 28), "\n")

class ContinuousNBClassfier:
    """Initialize a Naive Bayes Classifier for Continuous Mode

    Variables:
        psum: The sum of gray level of (label, pixel) = (i, j)
        psquaresum: The square sum of gray level of (label, pixel) = (i, j)
        pmean: The mean of gray level of (label, pixel) = (i, j)
        pvariance: The variance of gray level of (label, pixel) = (i, j)
        lfrequency: The number of occurence of label i
        lcount: The total number of labels
        lprobability: The probability of occurence of label i
    """
    def __init__(self):
        # 10 for labels: 0 ~ 9
        # 784 for features: 28x28 pixels
        self.psum = np.zeros(shape=(10, 784), dtype=float)
        self.psquaresum = np.zeros(shape=(10, 784), dtype=float)
        self.pmean = np.zeros(shape=(10, 784), dtype=float)
        self.pvariance = np.zeros(shape=(10, 784), dtype=float)
        self.lfrequency = np.zeros(shape=(10), dtype=int)
        self.lcount = 0
        self.lprobability = np.zeros(shape=(10), dtype=float)

    def train(self, Xtrain: Image, Ytrain: Lable, smoothing = 1000):
        nImage, nPixel = Xtrain.nImage, (Xtrain.nrow * Xtrain.ncol)
        self.lcount = Ytrain.nLable
        data = Xtrain.Images.reshape(nImage, nPixel).astype(float)

        # Update the sum, square sum of pixel and the frequency of label
        for idxImage in range(nImage):
            curlable = Ytrain.Lables[idxImage]
            self.lfrequency[curlable] += 1
            self.psum[curlable] += data[idxImage]
            self.psquaresum[curlable] += data[idxImage] ** 2

        # Calculate the probability of each lable
        self.lprobability = self.lfrequency / self.lcount

        # Calculate the mean(MLE) and variance(MLE) of pixel
        for idxlabel in range(10):
            self.pmean[idxlabel] = self.psum[idxlabel] / self.lfrequency[idxlabel]
            # Var[x] = E[x^2] - (E[x])^2
            self.pvariance[idxlabel] = self.psquaresum[idxlabel] / self.lfrequency[idxlabel] - self.pmean[idxlabel] ** 2 + smoothing

    def predict(self, Xtest: Image):
        nImage, nPixel = Xtest.nImage, (Xtest.nrow * Xtest.ncol)
        data = Xtest.Images.reshape(nImage, nPixel).astype(float)
        predictY = np.zeros(shape=(nImage), dtype=int)
        posterior = np.zeros(shape=(nImage, 10), dtype=float)

        for idxImage in range(nImage):
            for idxLable in range(10):
                # calculate likelohood, prior and posterior in log scale
                # probability of Gaussian distribution in log scale = -log(2 * pi * variance)/2 - (x-mean)^2 / (2*variance)
                likelihood = -np.log(2 * np.pi * self.pvariance[idxLable])/2 - ((data[idxImage] - self.pmean[idxLable]) ** 2) / (2 * self.pvariance[idxLable])
                prior = np.log(self.lprobability[idxLable])
                posterior[idxImage][idxLable] = np.sum(likelihood) + prior

            # marginalize posterior to let them sum up to 1
            posterior[idxImage] /= np.sum(posterior[idxImage])
            # the minimum posterior in log scale is the most probable
            predictY[idxImage] = np.argmin(posterior[idxImage])

        return predictY, posterior

    def calError(self, Ypredict, Ytest: Lable):
        nLable = Ytest.nLable
        wrong = np.sum(Ypredict != Ytest.Lables)
        return wrong / nLable

    def Imagination(self):
        imagination = (self.pmean >= 128).reshape(10, 28, 28).astype(np.uint8)
        print("Imagination of numbers in Bayesian classifier:\n")
        for idxLabel in range(10):
            print(idxLabel, ":")
            print(imagination[idxLabel], "\n")

def main(Args):
    ### Preprocessing ###
    toggle = Args.toggle
    Xtrain = readImage(TrainImagePath)
    Xtest = readImage(TestImagePath)
    Ytrain = readLable(TrainLablePath)
    Ytest = readLable(TestLablePath)
    ### Preprocessing END ###

    # Discrete Mode
    if toggle == 0:
        NBC = DiscreteNBClassifer()
    # Continuous mode
    elif toggle == 1:
        NBC = ContinuousNBClassfier()

    ### Main Part ###
    NBC.train(Xtrain, Ytrain)
    predictY, posterior = NBC.predict(Xtest)
    errorRate = NBC.calError(predictY, Ytest)

    ### Print the Result ###
    for cases in range(Ytest.nLable):
        print("Posterior (in log scale):")
        for label in range(10):
            print(label, ": ", posterior[cases][label], sep="")
        print("Prediction: ", predictY[cases], ", Ans: ", Ytest.Lables[cases], "\n", sep="")
    NBC.Imagination()
    print("Error rate:", errorRate)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('toggle', type=int, help="Please enter toggle option: 1 for discrete; 2 for continuous")
    args = parser.parse_args()
    main(args)

ML_2021/HW2 Naive Bayes & Online Learning/NaiveBayes.py