#!/usr/bin/env python
# -*- coding: utf-8 -*-

from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton, KeyboardButton, ReplyKeyboardMarkup, ReplyKeyboardRemove

account = KeyboardButton('ACCOUNT')
shop = KeyboardButton('SHOP')
policy = KeyboardButton('TERMS & CONDITIONS')
support = KeyboardButton('SUPPORT')

greet_kb = ReplyKeyboardMarkup(resize_keyboard=True, row_width=1)
greet_kb.add(account, shop, policy, support)

pay = InlineKeyboardButton('💰 PAY', callback_data='PAY')
btn_pay = InlineKeyboardMarkup(row_width=1).add(pay)

back_1 = InlineKeyboardButton('⬅ BACK', callback_data='BACK')
back_docs = InlineKeyboardButton('⬅ BACK', callback_data='BACK 2')

replenish = InlineKeyboardButton('💰 REPLENISH', callback_data='REPLENISH')
btn_replenish = InlineKeyboardMarkup(row_width=1).add(replenish)

сancel = InlineKeyboardButton('СANCEL', callback_data='СANCEL')
btn_сancel = InlineKeyboardMarkup(row_width=1).add(сancel)

a = InlineKeyboardButton('📰 DOCUMENTS', callback_data='DOCS')
b = InlineKeyboardButton('🔎 LOOK UP INFO', callback_data='LOOK UP INFO')
c = InlineKeyboardButton('🗓 FULL INFO', callback_data='FULL INFO')
d = InlineKeyboardButton('🕹 OWN REGISTERED BANK ACC.', callback_data='OWN REGISTERED BANK ACCOUNTS')
e = InlineKeyboardButton('💳 CC ENROLL', callback_data='CC ENROLL')
f = InlineKeyboardButton('💵 BRUT BANK ACCOUNTS', callback_data='BRUT BANK ACCOUNTS')
g = InlineKeyboardButton('📲 REAL USA NUMBER FOR SMS', callback_data='REAL USA NUMBER FOR SMS')

a_real_num = InlineKeyboardButton('ONE SMS - 5$', callback_data='ONE SMS - 5$')
real_num = InlineKeyboardMarkup(row_width=1).add(a_real_num)

h = InlineKeyboardButton('🇺🇸 US DOCUMENTS', callback_data='US DOCS')
j = InlineKeyboardButton('🇨🇦 CA DOCUMENTS', callback_data='CA DOCS')
l = InlineKeyboardButton('🇪🇺 EU DOCUMENTS', callback_data='EU DOCS')
k = InlineKeyboardButton('🌎 SOUTH AMERICA DOCUMENTS', callback_data='NORTH AMERICA DOCS')

a_ua_docs = InlineKeyboardButton('DL FRONT - 40$', callback_data='DL FRONT - 40$')
b_ua_docs = InlineKeyboardButton('DL FRONT & BACK - 60$', callback_data='DL FRONT & BACK - 60$')
c_ua_docs = InlineKeyboardButton('PASSPORT - 40$', callback_data='PASSPORT - 40$')
d_ua_docs = InlineKeyboardButton('SSN FRONT - 25$', callback_data='SSN FRONT - 25$')
e_ua_docs = InlineKeyboardButton('SSN FRONT & BACK - 40$', callback_data='SSN FRONT & BACK - 40$')
f_ua_docs = InlineKeyboardButton('UTILITY BILL - 30$', callback_data='UTILITY BILL - 30$')
g_ua_docs = InlineKeyboardButton('BANK STATEMENT - 35$', callback_data='BANK STATEMENT - 35$')
h_ua_docs = InlineKeyboardButton('DL 2 SIDE + SELFIE - 135$', callback_data='DL 2 SIDE + SELFIE - 135$')
j_ua_docs = InlineKeyboardButton('CC FRONT + BACK SIDE - 50$', callback_data='CC FRONT + BACK SIDE - 50$')
l_ua_docs = InlineKeyboardButton('TAX FROM 1040 & 1099 - 40$', callback_data='TAX FROM 1040 & 1099 - 40$')
k_ua_docs = InlineKeyboardButton('PAYSTUB - 40$', callback_data='PAYSTUB - 40$')

docs = InlineKeyboardMarkup(row_width=1).add(a, b, c, d, e, f, g)
docs_2 = InlineKeyboardMarkup(row_width=1).add(h, j, l, k, back_1)
us_docs = InlineKeyboardMarkup(row_width=1).add(a_ua_docs, b_ua_docs, c_ua_docs, d_ua_docs, e_ua_docs, f_ua_docs, g_ua_docs, h_ua_docs, j_ua_docs, l_ua_docs, k_ua_docs, back_docs)
ca_docs = InlineKeyboardMarkup(row_width=1).add(a_ua_docs, b_ua_docs, c_ua_docs, f_ua_docs, g_ua_docs, h_ua_docs, j_ua_docs, back_docs)
eu_docs = InlineKeyboardMarkup(row_width=1).add(a_ua_docs, b_ua_docs, c_ua_docs, f_ua_docs, g_ua_docs, h_ua_docs, j_ua_docs, back_docs)
na_docs = InlineKeyboardMarkup(row_width=1).add(a_ua_docs, b_ua_docs, c_ua_docs, f_ua_docs, g_ua_docs, h_ua_docs, j_ua_docs, back_docs)

a_full_info = InlineKeyboardButton('FULL INFO - 5$', callback_data='FULL INFO 2 - 5$')
b_full_info = InlineKeyboardButton('FULL INFO + DL - 17$', callback_data='FULL INFO + DL - 17$')
c_full_info = InlineKeyboardButton('FULL INFO CS 400-599 - 10$', callback_data='FULL INFO CS 400-599 - 10$')
d_full_info = InlineKeyboardButton('FULL INFO CS 600-699 - 15$', callback_data='FULL INFO CS 600-699 - 15$')
e_full_info = InlineKeyboardButton('FULL INFO CS 700-799 - 20$', callback_data='FULL INFO CS 700-799 - 20$')
f_full_info = InlineKeyboardButton('FULL INFO CS 800-899 - 45$', callback_data='FULL INFO CS 800-899 - 45$')
full_info = InlineKeyboardMarkup(row_width=1).add(a_full_info, b_full_info, c_full_info, d_full_info, e_full_info, f_full_info, back_1)

a_cc_enroll = InlineKeyboardButton('0.5-1k - 90$', callback_data='0.5-1k - 90$')
b_cc_enroll = InlineKeyboardButton('1k-3k - 150$', callback_data='1k-3k - 150$')
c_cc_enroll = InlineKeyboardButton('3k-5k - 200$', callback_data='3k-5k - 200$')
d_cc_enroll = InlineKeyboardButton('5k-10k - 400$', callback_data='5k-10k - 400$')
e_cc_enroll = InlineKeyboardButton('>10k - 550$', callback_data='>10k - 550$')
cc_enroll = InlineKeyboardMarkup(row_width=1).add(a_cc_enroll, b_cc_enroll, c_cc_enroll, d_cc_enroll, e_cc_enroll, back_1)

a_brut_bank_accounts = InlineKeyboardButton('0.5-1k - 50$', callback_data='bba 0.5-1k - 50$')
b_brut_bank_accounts = InlineKeyboardButton('1k-3k - 100$', callback_data='bba 1k-3k - 100$')
c_brut_bank_accounts = InlineKeyboardButton('3k-5k - 150$', callback_data='bba 3k-5k - 150$')
d_brut_bank_accounts = InlineKeyboardButton('5k-10k - 250$', callback_data='bba 5k-10k - 250$')
e_brut_bank_accounts = InlineKeyboardButton('>10k - 300$', callback_data='bba >10k - 300$')
brut_bank_accounts = InlineKeyboardMarkup(row_width=1).add(a_brut_bank_accounts, b_brut_bank_accounts, c_brut_bank_accounts, d_brut_bank_accounts, e_brut_bank_accounts, back_1)

a_own_reg_bank_acc = InlineKeyboardButton('CITI BANK - 75$', callback_data='CITI BANK - 75$')
b_own_reg_bank_acc = InlineKeyboardButton('BBT BANK - 75$', callback_data='BBT BANK - 75$')
c_own_reg_bank_acc = InlineKeyboardButton('CHASE BANK - 75$', callback_data='CHASE BANK - 75$')
d_own_reg_bank_acc = InlineKeyboardButton('TD BANK - 75$', callback_data='TD BANK - 75$')
e_own_reg_bank_acc = InlineKeyboardButton('CHIME BANK + VCC - 75$', callback_data='CHIME BANK + VCC - 75$')
own_reg_bank_acc = InlineKeyboardMarkup(row_width=1).add(a_own_reg_bank_acc, b_own_reg_bank_acc, c_own_reg_bank_acc, d_own_reg_bank_acc, e_own_reg_bank_acc, back_1)

a_look_up = InlineKeyboardButton('🇺🇸 US DOB + SNN - 20$', callback_data='US DOB + SNN - 20$')
b_look_up = InlineKeyboardButton('🇺🇸 US DRIVER LICENSE - 40$', callback_data='US DRIVER LICENSE - 40$')
c_look_up = InlineKeyboardButton('🇺🇸 US MMN - 35$', callback_data='US MMN - 35$')
d_look_up = InlineKeyboardButton('🇺🇸 US BACKGROUND REPORT - 25$', callback_data='US BACKGROUND REPORT - 25$')
e_look_up = InlineKeyboardButton('🇺🇸 US CREDIT REPORT - 35$', callback_data='US CREDIT REPORT - 35$')
f_look_up = InlineKeyboardButton('🇺🇸 US VEHICLE LOOK UP - 40$', callback_data='US VEHICLE LOOK UP - 40$')
g_look_up = InlineKeyboardButton('🇬🇧 UK SNN + DOB - 20$', callback_data='UK SNN + DOB  - 20$')
h_look_up = InlineKeyboardButton('SPECIAL ORDER', callback_data='SPECIAL ORDER')
look_up = InlineKeyboardMarkup(row_width=1).add(a_look_up, b_look_up, c_look_up, d_look_up, e_look_up, f_look_up, g_look_up, h_look_up, back_1)


import sys
import ctcsound
import time
import ctypes

orc_text = '''
  instr 1
	out(linen(oscili(p4,p5),0.1,p3,0.1))
	gkvar random 1, 2
  endin'''

sco_text = "i1 0 .5 1000 440"

cs = ctcsound.Csound()
cs.createMessageBuffer(False)

cs.setOption("-d")
cs.setOption("-odac")
cs.compileOrc(orc_text)
cs.readScore(sco_text)
cs.start()

while cs.performKsmps() == 0:
  string = cs.firstMessage()
  # Set the custom performance callback
  with open('/Users/j/Desktop/1.txt', 'a') as f:
    if string:
      f.write(string)
  cs.popFirstMessage()

cs.destroyMessageBuffer()
cs.cleanup()
cs.reset()


from Products.Five import BrowserView
from zope.component import getMultiAdapter
from Products.CMFCore.interfaces import ISiteRoot
from OFS.interfaces import IOrderedContainer
from plone.memoize import view
from cciaapd.contenttypes import logger

class SchedeContentView(BrowserView):

    def get_position_in_parent(self, obj):
        """
        Use IOrderedContainer interface to extract the object's manual ordering position
        """
        parent = obj.aq_inner.aq_parent
        ordered = IOrderedContainer(parent, None)
        if ordered is not None:
            return ordered.getObjectPosition(obj.getId())
        return 0

    def sort_by_position(self, a, b):
        """
        Python list sorter cmp() using position in parent.

        Descending order.
        """
        return self.get_position_in_parent(a) - self.get_position_in_parent(b)

    def get_results(self, portal_type):
        """Returns results depending on request and the following logic

        Case 1: a folder with a Scheda set as  default view ->
                return scheda contents filtered by content_types

        Case 2: the context is a Scheda ->
                return the contents of Moduli or Riferimenti according
                to the requeste portal type

        Case 3: as a fallback return an empty list
        """
        if ISiteRoot.providedBy(self.context):
            return []
        if self.has_default_view(self.context):
            return self.find_nephews(portal_type,
                                     self.get_default_view_object(self.context))

        if self.is_scheda(self.context):
            return self.find_nephews(portal_type)
        return []

    @view.memoize
    def has_default_view(self, item):
        """Return true if the default view is set to Scheda"""
        obj = self.get_default_view_object(item)
        if not obj:
            return False
        return obj.portal_type == 'Scheda'

    @view.memoize
    def get_default_view_object(self, item):
        ''' Check for default page and get it from the context
        if default page is not set return None, otherwise the object
        '''
        default_page_method = getattr(item, 'getDefaultPage', None)
        if not default_page_method:
            return None
        default_page = default_page_method()
        obj = item.get(default_page)
        return obj

    def find_nephews(self, portal_type, context=None):
        '''
        Check inside objects all the filtered types and
        return their children.
        Moreover, parent folders must be traversed to get contents of the
        first one where the default view is set as Scheda, if exists.
        '''
        if context is None:
            context = self.context
        results = []
        schede_deep = 0
        for item in context.aq_chain:
            if schede_deep == 2:
                break
            context_state = getMultiAdapter(
                (item, self.request),
                name=u'plone_context_state'
            )
            if context_state.is_default_page():
                # if is a default view, skip it, we'll handle it on the folder.
                # this avoid double entries
                continue
            if not self.is_scheda(item) and not self.has_default_view(item):
                # if the context isn't a scheda and its default_view isn't a
                # scheda, skip it
                continue
            schede_attachments = self.get_scheda_attachments(item, portal_type)
            if schede_attachments:
                schede_deep = schede_deep + 1
                results.extend(schede_attachments)
        return results

    def get_scheda_attachments(self, scheda, portal_type):
        default_view_obj = self.get_default_view_object(scheda)
        if default_view_obj and self.is_scheda(default_view_obj):
            scheda = default_view_obj
        children = scheda.listFolderContents(
            contentFilter={'portal_type': portal_type})
        results = []
        for child in children:
            items = [x for x in child.listFolderContents() if not x.isExpired()]
            if items:
                results.extend(sorted(items, self.sort_by_position))
        return results

    def is_scheda(self, context):
        """Return true if the content is a Scheda"""
        return getattr(context, 'portal_type', "") == "Scheda"


#!/bin/env python
import ROOT
import sys
sys.path.append('..')
from collections import namedtuple

from objects.MultiGraph   import MultiGraph
from objects.FitFunctions import crystalballEfficiency
from objects.Fitter       import Fitter
from plot.CMSStyle        import CMS_lumi
from plot.PlotStyle       import PlotStyle

ROOT.TH1.SetDefaultSumw2()
ROOT.gROOT.SetBatch(True)

ROOT.gStyle.SetLegendFont(42)

ROOT.gStyle.SetOptStat(False)

ROOT.gStyle.SetOptFit(000000)

c1 = ROOT.TCanvas('', '', 700, 700)

ROOT.gPad.SetGridx()
ROOT.gPad.SetGridy()

func = ROOT.TF1('func', crystalballEfficiency, 0., 200., 5)
func.SetNpx(100000)
func.SetParameter(0,  37.5 )
func.SetParameter(1,   2.3 )
func.SetParameter(2,   1.4  )
func.SetParameter(3,   3.   )
func.SetParameter(4,   0.96)

func.SetParLimits(0,  10.  ,  50.  )
func.SetParLimits(1,   0.5 ,  30.  )
func.SetParLimits(2,   0.01,  40.  )
# func.SetParLimits(3,   1.01, 999.  )
func.SetParLimits(3,  1.01, 141. )
func.SetParLimits(4,  0.5, 1. )

# func.FixParameter(0, 37. )
# func.FixParameter(1, 7.  )
# func.FixParameter(2, 0.92 )
# func.FixParameter(3, 139. )
# func.FixParameter(4, 0.94 )

func.SetParName(0, 'm_{0}')
func.SetParName(1, 'sigma')
func.SetParName(2, 'alpha')
func.SetParName(3, 'n'    )
func.SetParName(4, 'norm' )

# func = ROOT.TF1('func', '[0]*TMath::Erf((x-[1])/[2])', 0., 180.)  # non def pos erf
# func = ROOT.TF1('func', '0.5 * ([0]*TMath::Erf((x-[1])/[2]) + 1)', 0., 180.) # def pos erf
# func.SetParameter(0,  0.9)
# func.SetParameter(1, 20. )
# func.SetParameter(2, 10. )
#
# func.SetParName(0, 'plateau')
# func.SetParName(1, 'threshold')
# func.SetParName(2, 'width')

myGraph = namedtuple('mygraph', 'file graph function colour legend legendOpt')

# file = '../test/diTau_plots.root'
file = '../test/diTau_plots_good.root'
# file = '../test/diTau_plots_high_mt_os.root'
# file = '../test/diTau_plots_ss.root'
# file = '../test/mu_tau_singleL1_plots.root'
# file = '../test/mu_tau_singleL1_plots_high_mt_os.root'
# file = '../test/mu_tau_singleL1_plots_ss.root'
# file = '../test/mu_tau_crossL1_plots.root'
# file = '../test/mu_tau_crossL1_plots_high_mt_os.root'
# file = '../test/mu_tau_crossL1_plots_ss.root'

graphs = [
    myGraph(file=file, graph='lowmt_zmass_sub_NoIso_HLT/tau_pt'      , function=func, colour=ROOT.kOrange-6, legend='NoIso'     , legendOpt='EP'),
    myGraph(file=file, graph='lowmt_zmass_sub_VLooseIso_HLT/tau_pt'  , function=func, colour=ROOT.kOrange-4, legend='VLooseIso' , legendOpt='EP'),
    myGraph(file=file, graph='lowmt_zmass_sub_LooseIso_HLT/tau_pt'   , function=func, colour=ROOT.kOrange-2, legend='LooseIso'  , legendOpt='EP'),
    myGraph(file=file, graph='lowmt_zmass_sub_MediumIso_HLT/tau_pt'  , function=func, colour=ROOT.kOrange  , legend='MediumIso' , legendOpt='EP'),
    myGraph(file=file, graph='lowmt_zmass_sub_TightIso_HLT/tau_pt'   , function=func, colour=ROOT.kOrange+2, legend='TightIso'  , legendOpt='EP'),
    myGraph(file=file, graph='lowmt_zmass_sub_VTightIso_HLT/tau_pt'  , function=func, colour=ROOT.kOrange+4, legend='VTightIso' , legendOpt='EP'),
    myGraph(file=file, graph='lowmt_zmass_sub_VVTightIso_HLT/tau_pt' , function=func, colour=ROOT.kOrange+6, legend='VVTightIso', legendOpt='EP'),
]

# graphs = [
#     myGraph(file=file, graph='highmt_os_NoIso_HLT/tau_pt'      , function=func, colour=ROOT.kAzure-6, legend='NoIso'     , legendOpt='EP'),
#     myGraph(file=file, graph='highmt_os_VLooseIso_HLT/tau_pt'  , function=func, colour=ROOT.kAzure-4, legend='VLooseIso' , legendOpt='EP'),
#     myGraph(file=file, graph='highmt_os_LooseIso_HLT/tau_pt'   , function=func, colour=ROOT.kAzure-2, legend='LooseIso'  , legendOpt='EP'),
#     myGraph(file=file, graph='highmt_os_MediumIso_HLT/tau_pt'  , function=func, colour=ROOT.kAzure  , legend='MediumIso' , legendOpt='EP'),
#     myGraph(file=file, graph='highmt_os_TightIso_HLT/tau_pt'   , function=func, colour=ROOT.kAzure+2, legend='TightIso'  , legendOpt='EP'),
#     myGraph(file=file, graph='highmt_os_VTightIso_HLT/tau_pt'  , function=func, colour=ROOT.kAzure+4, legend='VTightIso' , legendOpt='EP'),
#     myGraph(file=file, graph='highmt_os_VVTightIso_HLT/tau_pt' , function=func, colour=ROOT.kAzure+6, legend='VVTightIso', legendOpt='EP'),
# ]

# graphs = [
#     myGraph(file=file, graph='ss_NoIso_HLT/tau_pt'      , function=func, colour=ROOT.kViolet-6, legend='NoIso'     , legendOpt='EP'),
#     myGraph(file=file, graph='ss_VLooseIso_HLT/tau_pt'  , function=func, colour=ROOT.kViolet-4, legend='VLooseIso' , legendOpt='EP'),
#     myGraph(file=file, graph='ss_LooseIso_HLT/tau_pt'   , function=func, colour=ROOT.kViolet-2, legend='LooseIso'  , legendOpt='EP'),
#     myGraph(file=file, graph='ss_MediumIso_HLT/tau_pt'  , function=func, colour=ROOT.kViolet  , legend='MediumIso' , legendOpt='EP'),
#     myGraph(file=file, graph='ss_TightIso_HLT/tau_pt'   , function=func, colour=ROOT.kViolet+2, legend='TightIso'  , legendOpt='EP'),
#     myGraph(file=file, graph='ss_VTightIso_HLT/tau_pt'  , function=func, colour=ROOT.kViolet+4, legend='VTightIso' , legendOpt='EP'),
#     myGraph(file=file, graph='ss_VVTightIso_HLT/tau_pt' , function=func, colour=ROOT.kViolet+6, legend='VVTightIso', legendOpt='EP'),
# ]

mg = MultiGraph(graphs=graphs)
mg.Fill()
mg.Draw('AP')

mg.GetYaxis().SetTitle('L1 + HLT efficiency')
mg.GetXaxis().SetTitle('offline #tau p_{T} [GeV]')
mg.GetYaxis().SetTitleOffset(1.3)
mg.GetXaxis().SetTitleOffset(1.3)

mg.GetXaxis().SetRangeUser(0., 170.)
mg.GetYaxis().SetRangeUser(0., 1.05)

# ROOT.gPad.SetLogx()
mg.leg.Draw('SAMEAPZL')

ROOT.gPad.Update()
CMS_lumi(ROOT.gPad, 4, 0)

c1.SaveAs('L1efficiencyDataMC.pdf')


import re
import json
import os
import datetime

cwd = os.getcwd()

START_DATE = datetime.datetime(2022, 12, 26, 0, 0)

def get_wjc_day(today):
  return (today - START_DATE).days

# Input format is
#     FirstInitial. Lastname, SOG, M, SS, ...
def time_to_float(x):
  return round(x/60, 2)

def write_to_data(manual_player_data, date):
  # Make a json with updated reference to json
  with open(cwd + '/json/beta-wjc/2022-23/manual-player-data.json', 'w') as json_file:
    json_file.write(json.dumps(manual_player_data, indent=4))
  return

def parse_manual_data(matches, date):
  # manual-player-data.json data for reference
  manual_player_data = {}
  with open(cwd + '/json/beta-wjc/2022-23/manual-player-data.json', 'r') as json_file:
    manual_player_data = json.loads(json_file.read())
    # name-lookup-table.json translates short hand name to reference name
    with open(cwd + '/json/beta-wjc/2022-23/name-lookup-table.json', 'r') as json_file_2:
      name_lookup_table = json.loads(json_file_2.read())
      #print(f"\t### Manual Player Data: {manual_player_data}")

      # Pull data from list and update reference to json
      for id in matches:
        # id format is:
        #     FirstInitial. Lastname, SOG, M, SS, ...
        # name, SOG, min, sec = re.match('([\w|\s]*),([0-9]+),([0-9]{2}),([0-5][0-9]?)', id).groups()
        name, SOG, min, sec = re.match('[\s]*([\w|\s|.|\']*),[\s]*([0-9]+)[\s]*,[\s]*([0-9]{1,2})[\s]*,[\s]*([0-5][0-9]{0,1})[\s]*?', id).groups()
        name = name.rstrip()

        TPM = float(min) + time_to_float(float(sec))

        found = False
        for team in name_lookup_table:
          for player in name_lookup_table[team]:
            found = False
            if player.lower() == name.lower():
              name = name_lookup_table[team][player]
              #print(f"\t Player:{player}, Name:{name}")
              #print(f"\tFound manual player data: {manual_player_data[team][name]["SOG"]}, {manual_player_data[team][name]["TPM"]}, {int(SOG)}, {TPM}")
              manual_player_data[team][name]["SOG"] += int(SOG)
              manual_player_data[team][name]["TPM"] = round(manual_player_data[team][name]["TPM"] + TPM, 2)
              found = True
              break

          if found:
            break

  write_to_data(manual_player_data, date)
  return

def main():

  date = str(get_wjc_day(datetime.datetime.today()))

  # Prompt user to input list
  data_list = input('Input, ex FirstInitial. Lastname, SOG, M, SS, ...: ')

  matches = re.findall('[\w|\s|.|\']*,[\s]*[0-9]+[\s]*,[\s]*[0-9]{1,2}[\s]*,[\s]*[0-5][0-9]{0,1}[\s]*?', data_list)

  parse_manual_data(matches, date)

  print(f'''
  Manual player data has been updated and written to /json/beta-wjc/manual-player-data.json
  Run `python src/beta-wjc/merge-data.py` to update the data in /json/beta-wjc/2022-2023/manual-player-data.json
  ''')

if __name__ == "__main__":
  main()


import inspect
import time

from django.http import HttpResponse, JsonResponse
from django.core import serializers
from django.shortcuts import render
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import ensure_csrf_cookie

from .models import Transaction
from user_group.models import User

# Create your views here.
# @method_decorator(ensure_csrf_cookie)
def createTransaction(request):
    if request.method == "GET":
        transactions = Transaction.objects.all().values()
        return JsonResponse(list(transactions), safe=False)

    if request.method == "POST":
        userid = int(request.POST['user'])
        amount = float(request.POST['amount'])
        owner = User.objects.get(pk=userid)
        Transaction.objects.create(owner=owner, amount=amount)

        return HttpResponse("Hello from POST " + inspect.stack()[0][3])

# @method_decorator(ensure_csrf_cookie)
def updateTransaction(request):
    if request.method == "POST":
        transactionid = int(request.POST['transactionid'])
        transaction = Transaction.objects.get(pk=transactionid)
        amount = transaction.amount
        transaction.amount = amount + 100
        time.sleep(0.005)
        transaction.save()

        return JsonResponse(serializers.serialize("json",[transaction,]), safe=False)

    if request.method == "GET":
        return HttpResponse("Hello from Get " + inspect.stack()[0][3])

def restoreTransaction(request):
    if request.method == "POST":
        transactionid = int(request.POST['transactionid'])
        transaction = Transaction.objects.get(pk=transactionid)
        amount = transaction.amount
        time.sleep(0.007)
        transaction.amount = amount - 100
        transaction.save()

        return JsonResponse(serializers.serialize("json",[transaction,]), safe=False)

    if request.method == "GET":
        return HttpResponse("Hello from Get " + inspect.stack()[0][3])

def resetTransaction(request):
    if request.method == "POST":
        transactionid = int(request.POST['transactionid'])
        transaction = Transaction.objects.get(pk=transactionid)
        amount = transaction.amount
        transaction.amount = 0
        transaction.save()

        return JsonResponse(serializers.serialize("json",[transaction,]), safe=False)

    if request.method == "GET":
        return HttpResponse("Hello from Get " + inspect.stack()[0][3])

# @method_decorator(ensure_csrf_cookie)
def getTransactions(request):
    if request.method == "POST":
        return HttpResponse("Hello from Post " + inspect.stack()[0][3])

    if request.method == "GET":
        return HttpResponse("Hello from Get " + inspect.stack()[0][3])


from collections import deque

def solution(priorities, location):
    # 프로세스의 위치를 담아주고
    pro = [i for i in range(len(priorities))]
    # 프로세스의 번호마다 실행한 순서를 기록할 딕셔너리
    p_dic = {i:-1 for i in range(len(priorities))}
    # 이 두가지를 모두 deque의 queue로 지정한다.
    queue = deque(priorities)
    p_queue = deque(pro)
    order = 1
    # queue가 모두 실행할 때 까지 반복하며
    while queue:
        # queue의 max값이 queue의 첫번째면
        if max(queue) == queue[0]:
            # queue를 삭재하고
            queue.popleft()
            # 해당 queue의 실행 순서를 넣고
            p_dic[p_queue.popleft()] = order
            # 순서를 + 1
            order += 1
        # max값이 아니라면
        else:
            # queue값과 프로세스의 위치를 popleft하고
            pri = queue.popleft()
            num = p_queue.popleft()
            # 다시 맨 뒤에 삽입한다.
            queue.append(pri)
            p_queue.append(num)

    # 마지막으로 알고자 하는 프로세스의 실행순서를 반환.
    return p_dic[location]


"""
Tests for HTTP storage client + server.

The tests here are synchronous and don't involve running a real reactor.  This
works, but has some caveats when it comes to testing HTTP endpoints:

* Some HTTP endpoints are synchronous, some are not.
* For synchronous endpoints, the result is immediately available on the
  ``Deferred`` coming out of ``StubTreq``.
* For asynchronous endpoints, you need to use ``StubTreq.flush()`` and
  iterate the fake in-memory clock/reactor to advance time .

So for HTTP endpoints, you should use ``HttpTestFixture.result_of_with_flush()``
which handles both, and patches and moves forward the global Twisted
``Cooperator`` since that is used to drive pull producers. This is,
sadly, an internal implementation detail of Twisted being leaked to tests...

For definitely synchronous calls, you can just use ``result_of()``.
"""

import time
from base64 import b64encode
from contextlib import contextmanager
from os import urandom
from typing import Union, Callable, Tuple, Iterable
from queue import Queue
from cbor2 import dumps
from pycddl import ValidationError as CDDLValidationError
from hypothesis import assume, given, strategies as st, settings as hypothesis_settings
from fixtures import Fixture, TempDir, MonkeyPatch
from treq.testing import StubTreq
from klein import Klein
from hyperlink import DecodedURL
from collections_extended import RangeMap
from twisted.internet.task import Clock, Cooperator
from twisted.internet.interfaces import IReactorTime, IReactorFromThreads
from twisted.internet.defer import CancelledError, Deferred, ensureDeferred
from twisted.web import http
from twisted.web.http_headers import Headers
from werkzeug import routing
from werkzeug.exceptions import NotFound as WNotFound
from testtools.matchers import Equals
from zope.interface import implementer

from ..util.deferredutil import async_to_deferred
from ..util.cputhreadpool import disable_thread_pool_for_test
from .common import SyncTestCase
from ..storage.http_common import (
    get_content_type,
    CBOR_MIME_TYPE,
    response_is_not_html,
)
from ..storage.common import si_b2a
from ..storage.lease import LeaseInfo
from ..storage.server import StorageServer
from ..storage.http_server import (
    HTTPServer,
    _extract_secrets,
    Secrets,
    ClientSecretsException,
    _authorized_route,
    StorageIndexConverter,
    _add_error_handling,
    read_encoded,
    _SCHEMAS as SERVER_SCHEMAS,
    BaseApp,
)
from ..storage.http_client import (
    StorageClient,
    StorageClientFactory,
    ClientException,
    StorageClientImmutables,
    ImmutableCreateResult,
    UploadProgress,
    StorageClientGeneral,
    _encode_si,
    StorageClientMutables,
    TestWriteVectors,
    WriteVector,
    ReadVector,
    ReadTestWriteResult,
    TestVector,
    limited_content,
)

class HTTPUtilities(SyncTestCase):
    """Tests for HTTP common utilities."""

    def test_get_content_type(self):
        """``get_content_type()`` extracts the content-type from the header."""

        def assert_header_values_result(values, expected_content_type):
            headers = Headers()
            if values:
                headers.setRawHeaders("Content-Type", values)
            content_type = get_content_type(headers)
            self.assertEqual(content_type, expected_content_type)

        assert_header_values_result(["text/html"], "text/html")
        assert_header_values_result([], None)
        assert_header_values_result(["text/plain", "application/json"], "text/plain")
        assert_header_values_result(["text/html;encoding=utf-8"], "text/html")

def _post_process(params):
    secret_types, secrets = params
    secrets = {t: s for (t, s) in zip(secret_types, secrets)}
    headers = [
        "{} {}".format(
            secret_type.value, str(b64encode(secrets[secret_type]), "ascii").strip()
        )
        for secret_type in secret_types
    ]
    return secrets, headers

# Creates a tuple of ({Secret enum value: secret_bytes}, [http headers with secrets]).
SECRETS_STRATEGY = (
    st.sets(st.sampled_from(Secrets))
    .flatmap(
        lambda secret_types: st.tuples(
            st.just(secret_types),
            st.lists(
                st.binary(min_size=32, max_size=32),
                min_size=len(secret_types),
                max_size=len(secret_types),
            ),
        )
    )
    .map(_post_process)
)

class ExtractSecretsTests(SyncTestCase):
    """
    Tests for ``_extract_secrets``.
    """

    @given(secrets_to_send=SECRETS_STRATEGY)
    def test_extract_secrets(self, secrets_to_send):
        """
        ``_extract_secrets()`` returns a dictionary with the extracted secrets
        if the input secrets match the required secrets.
        """
        secrets, headers = secrets_to_send

        # No secrets needed, none given:
        self.assertEqual(_extract_secrets(headers, secrets.keys()), secrets)

    @given(
        secrets_to_send=SECRETS_STRATEGY,
        secrets_to_require=st.sets(st.sampled_from(Secrets)),
    )
    def test_wrong_number_of_secrets(self, secrets_to_send, secrets_to_require):
        """
        If the wrong number of secrets are passed to ``_extract_secrets``, a
        ``ClientSecretsException`` is raised.
        """
        secrets_to_send, headers = secrets_to_send
        assume(secrets_to_send.keys() != secrets_to_require)

        with self.assertRaises(ClientSecretsException):
            _extract_secrets(headers, secrets_to_require)

    def test_bad_secret_missing_value(self):
        """
        Missing value in ``_extract_secrets`` result in
        ``ClientSecretsException``.
        """
        with self.assertRaises(ClientSecretsException):
            _extract_secrets(["lease-renew-secret"], {Secrets.LEASE_RENEW})

    def test_bad_secret_unknown_prefix(self):
        """
        Missing value in ``_extract_secrets`` result in
        ``ClientSecretsException``.
        """
        with self.assertRaises(ClientSecretsException):
            _extract_secrets(["FOO eA=="], set())

    def test_bad_secret_not_base64(self):
        """
        A non-base64 value in ``_extract_secrets`` result in
        ``ClientSecretsException``.
        """
        with self.assertRaises(ClientSecretsException):
            _extract_secrets(["lease-renew-secret x"], {Secrets.LEASE_RENEW})

    def test_bad_secret_wrong_length_lease_renew(self):
        """
        Lease renewal secrets must be 32-bytes long.
        """
        with self.assertRaises(ClientSecretsException):
            _extract_secrets(["lease-renew-secret eA=="], {Secrets.LEASE_RENEW})

    def test_bad_secret_wrong_length_lease_cancel(self):
        """
        Lease cancel secrets must be 32-bytes long.
        """
        with self.assertRaises(ClientSecretsException):
            _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW})

class RouteConverterTests(SyncTestCase):
    """Tests for custom werkzeug path segment converters."""

    adapter = routing.Map(
        [
            routing.Rule(
                "/<storage_index:storage_index>/", endpoint="si", methods=["GET"]
            )
        ],
        converters={"storage_index": StorageIndexConverter},
    ).bind("example.com", "/")

    @given(storage_index=st.binary(min_size=16, max_size=16))
    def test_good_storage_index_is_parsed(self, storage_index):
        """
        A valid storage index is accepted and parsed back out by
        StorageIndexConverter.
        """
        self.assertEqual(
            self.adapter.match(
                "/{}/".format(str(si_b2a(storage_index), "ascii")), method="GET"
            ),
            ("si", {"storage_index": storage_index}),
        )

    def test_long_storage_index_is_not_parsed(self):
        """An overly long storage_index string is not parsed."""
        with self.assertRaises(WNotFound):
            self.adapter.match("/{}/".format("a" * 27), method="GET")

    def test_short_storage_index_is_not_parsed(self):
        """An overly short storage_index string is not parsed."""
        with self.assertRaises(WNotFound):
            self.adapter.match("/{}/".format("a" * 25), method="GET")

    def test_bad_characters_storage_index_is_not_parsed(self):
        """A storage_index string with bad characters is not parsed."""
        with self.assertRaises(WNotFound):
            self.adapter.match("/{}_/".format("a" * 25), method="GET")

    def test_invalid_storage_index_is_not_parsed(self):
        """An invalid storage_index string is not parsed."""
        with self.assertRaises(WNotFound):
            self.adapter.match("/nomd2a65ylxjbqzsw7gcfh4ivr/", method="GET")

# TODO should be actual swissnum
SWISSNUM_FOR_TEST = b"abcd"

def gen_bytes(length: int) -> bytes:
    """Generate bytes to the given length."""
    result = (b"0123456789abcdef" * ((length // 16) + 1))[:length]
    assert len(result) == length
    return result

class TestApp(BaseApp):
    """HTTP API for testing purposes."""

    clock: IReactorTime
    _app = Klein()
    _add_error_handling(_app)
    _swissnum = SWISSNUM_FOR_TEST  # Match what the test client is using

    @_authorized_route(_app, set(), "/noop", methods=["GET"])
    def noop(self, request, authorization):
        return "noop"

    @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"])
    def validate_upload_secret(self, request, authorization):
        if authorization == {Secrets.UPLOAD: b"MAGIC"}:
            return "GOOD SECRET"
        else:
            return "BAD: {}".format(authorization)

    @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"])
    def bad_version(self, request, authorization):
        """Return version result that violates the expected schema."""
        request.setHeader("content-type", CBOR_MIME_TYPE)
        return dumps({"garbage": 123})

    @_authorized_route(_app, set(), "/bytes/<int:length>", methods=["GET"])
    def generate_bytes(self, request, authorization, length):
        """Return bytes to the given length using ``gen_bytes()``."""
        return gen_bytes(length)

    @_authorized_route(_app, set(), "/slowly_never_finish_result", methods=["GET"])
    def slowly_never_finish_result(self, request, authorization):
        """
        Send data immediately, after 59 seconds, after another 59 seconds, and then
        never again, without finishing the response.
        """
        request.write(b"a")
        self.clock.callLater(59, request.write, b"b")
        self.clock.callLater(59 + 59, request.write, b"c")
        return Deferred()

    @_authorized_route(_app, set(), "/die_unfinished", methods=["GET"])
    def die(self, request, authorization):
        """
        Dies half-way.
        """
        request.transport.loseConnection()
        return Deferred()

    @_authorized_route(_app, set(), "/read_body", methods=["POST"])
    @async_to_deferred
    async def read_body(self, request, authorization):
        """
        Accept an advise_corrupt_share message, return the reason.

        I.e. exercise codepaths used for reading CBOR from the body.
        """
        data = await read_encoded(
            self.clock, request, SERVER_SCHEMAS["advise_corrupt_share"]
        )
        return data["reason"]

def result_of(d):
    """
    Synchronously extract the result of a Deferred.
    """
    result = []
    error = []
    d.addCallbacks(result.append, error.append)
    if result:
        return result[0]
    if error:
        error[0].raiseException()
    raise RuntimeError(
        "We expected given Deferred to have result already, but it wasn't. "
        + "This is probably a test design issue."
    )

class CustomHTTPServerTests(SyncTestCase):
    """
    Tests that use a custom HTTP server.
    """

    def setUp(self):
        super(CustomHTTPServerTests, self).setUp()
        disable_thread_pool_for_test(self)
        StorageClientFactory.start_test_mode(
            lambda pool: self.addCleanup(pool.closeCachedConnections)
        )
        self.addCleanup(StorageClientFactory.stop_test_mode)
        # Could be a fixture, but will only be used in this test class so not
        # going to bother:
        self._http_server = TestApp()
        treq = StubTreq(self._http_server._app.resource())
        self.client = StorageClient(
            DecodedURL.from_text("http://127.0.0.1"),
            SWISSNUM_FOR_TEST,
            treq=treq,
            pool=None,
            # We're using a Treq private API to get the reactor, alas, but only
            # in a test, so not going to worry about it too much. This would be
            # fixed if https://github.com/twisted/treq/issues/226 were ever
            # fixed.
            clock=treq._agent._memoryReactor,
            analyze_response=response_is_not_html,
        )
        self._http_server.clock = self.client._clock

    def test_bad_swissnum_from_client(self) -> None:
        """
        If the swissnum is invalid, a BAD REQUEST response code is returned.
        """
        headers = Headers()
        # The value is not UTF-8.
        headers.addRawHeader("Authorization", b"\x00\xFF\x00\xFF")
        response = result_of(
            self.client._treq.request(
                "GET",
                DecodedURL.from_text("http://127.0.0.1/noop"),
                headers=headers,
            )
        )
        self.assertEqual(response.code, 400)

    def test_bad_secret(self) -> None:
        """
        If the secret is invalid (not base64), a BAD REQUEST
        response code is returned.
        """
        bad_secret = b"upload-secret []<>"
        headers = Headers()
        headers.addRawHeader(
            "X-Tahoe-Authorization",
            bad_secret,
        )
        response = result_of(
            self.client.request(
                "GET",
                DecodedURL.from_text("http://127.0.0.1/upload_secret"),
                headers=headers,
            )
        )
        self.assertEqual(response.code, 400)

    def test_authorization_enforcement(self):
        """
        The requirement for secrets is enforced by the ``_authorized_route``
        decorator; if they are not given, a 400 response code is returned.

        Note that this refers to ``X-Tahoe-Authorization``, not the
        ``Authorization`` header used for the swissnum.
        """
        # Without secret, get a 400 error.
        response = result_of(
            self.client.request(
                "GET",
                DecodedURL.from_text("http://127.0.0.1/upload_secret"),
            )
        )
        self.assertEqual(response.code, 400)

        # With secret, we're good.
        response = result_of(
            self.client.request(
                "GET",
                DecodedURL.from_text("http://127.0.0.1/upload_secret"),
                upload_secret=b"MAGIC",
            )
        )
        self.assertEqual(response.code, 200)
        self.assertEqual(result_of(response.content()), b"GOOD SECRET")

    def test_client_side_schema_validation(self):
        """
        The client validates returned CBOR message against a schema.
        """
        client = StorageClientGeneral(self.client)
        with self.assertRaises(CDDLValidationError):
            result_of(client.get_version())

    @given(length=st.integers(min_value=1, max_value=1_000_000))
    # On Python 3.12 we're getting weird deadline issues in CI, so disabling
    # for now.
    @hypothesis_settings(deadline=None)
    def test_limited_content_fits(self, length):
        """
        ``http_client.limited_content()`` returns the body if it is less than
        the max length.
        """
        for at_least_length in (length, length + 1, length + 1000, length + 100_000):
            response = result_of(
                self.client.request(
                    "GET",
                    DecodedURL.from_text(f"http://127.0.0.1/bytes/{length}"),
                )
            )

            self.assertEqual(
                result_of(
                    limited_content(response, self._http_server.clock, at_least_length)
                ).read(),
                gen_bytes(length),
            )

    @given(length=st.integers(min_value=10, max_value=1_000_000))
    def test_limited_content_does_not_fit(self, length):
        """
        If the body is longer than than max length,
        ``http_client.limited_content()`` fails with a ``ValueError``.
        """
        for too_short in (length - 1, 5):
            response = result_of(
                self.client.request(
                    "GET",
                    DecodedURL.from_text(f"http://127.0.0.1/bytes/{length}"),
                )
            )

            with self.assertRaises(ValueError):
                result_of(limited_content(response, self._http_server.clock, too_short))

    def test_limited_content_silence_causes_timeout(self):
        """
        ``http_client.limited_content() times out if it receives no data for 60
        seconds.
        """
        response = result_of(
            self.client.request(
                "GET",
                DecodedURL.from_text("http://127.0.0.1/slowly_never_finish_result"),
            )
        )

        body_deferred = limited_content(response, self._http_server.clock, 4)
        result = []
        error = []
        body_deferred.addCallbacks(result.append, error.append)

        for i in range(59 + 59 + 60):
            self.assertEqual((result, error), ([], []))
            self._http_server.clock.advance(1)
            # Push data between in-memory client and in-memory server:
            self.client._treq._agent.flush()

        # After 59 (second write) + 59 (third write) + 60 seconds (quiescent
        # timeout) the limited_content() response times out.
        self.assertTrue(error)
        with self.assertRaises(CancelledError):
            error[0].raiseException()

    def test_limited_content_cancels_timeout_on_failed_response(self):
        """
        If the response fails somehow, the timeout is still cancelled.
        """
        response = result_of(
            self.client.request(
                "GET",
                DecodedURL.from_text("http://127.0.0.1/die"),
            )
        )

        d = limited_content(response, self._http_server.clock, 4)
        with self.assertRaises(ValueError):
            result_of(d)
        self.assertEqual(len(self._http_server.clock.getDelayedCalls()), 0)

    def test_request_with_no_content_type_same_as_cbor(self):
        """
        If no ``Content-Type`` header is set when sending a body, it is assumed
        to be CBOR.
        """
        response = result_of(
            self.client.request(
                "POST",
                DecodedURL.from_text("http://127.0.0.1/read_body"),
                data=dumps({"reason": "test"}),
            )
        )
        self.assertEqual(
            result_of(limited_content(response, self._http_server.clock, 100)).read(),
            b"test",
        )

    def test_request_with_wrong_content(self):
        """
        If a non-CBOR ``Content-Type`` header is set when sending a body, the
        server complains appropriatly.
        """
        headers = Headers()
        headers.setRawHeaders("content-type", ["some/value"])
        response = result_of(
            self.client.request(
                "POST",
                DecodedURL.from_text("http://127.0.0.1/read_body"),
                data=dumps({"reason": "test"}),
                headers=headers,
            )
        )
        self.assertEqual(response.code, http.UNSUPPORTED_MEDIA_TYPE)

@implementer(IReactorFromThreads)
class Reactor(Clock):
    """
    Fake reactor that supports time APIs and callFromThread.

    Advancing the clock also runs any callbacks scheduled via callFromThread.
    """

    def __init__(self):
        Clock.__init__(self)
        self._queue = Queue()

    def callFromThread(self, callable, *args, **kwargs):
        self._queue.put((callable, args, kwargs))

    def advance(self, *args, **kwargs):
        Clock.advance(self, *args, **kwargs)
        while not self._queue.empty():
            f, args, kwargs = self._queue.get()
            f(*args, **kwargs)

class HttpTestFixture(Fixture):
    """
    Setup HTTP tests' infrastructure, the storage server and corresponding
    client.
    """

    def _setUp(self):
        StorageClientFactory.start_test_mode(
            lambda pool: self.addCleanup(pool.closeCachedConnections)
        )
        self.addCleanup(StorageClientFactory.stop_test_mode)
        self.clock = Reactor()
        self.tempdir = self.useFixture(TempDir())
        # The global Cooperator used by Twisted (a) used by pull producers in
        # twisted.web, (b) is driven by a real reactor. We want to push time
        # forward ourselves since we rely on pull producers in the HTTP storage
        # server.
        self.mock = self.useFixture(
            MonkeyPatch(
                "twisted.internet.task._theCooperator",
                Cooperator(scheduler=lambda c: self.clock.callLater(0.000001, c)),
            )
        )
        self.storage_server = StorageServer(
            self.tempdir.path, b"\x00" * 20, clock=self.clock
        )
        self.http_server = HTTPServer(
            self.clock, self.storage_server, SWISSNUM_FOR_TEST
        )
        self.treq = StubTreq(self.http_server.get_resource())
        self.client = StorageClient(
            DecodedURL.from_text("http://127.0.0.1"),
            SWISSNUM_FOR_TEST,
            treq=self.treq,
            pool=None,
            clock=self.clock,
            analyze_response=response_is_not_html,
        )

    def result_of_with_flush(self, d):
        """
        Like ``result_of``, but supports fake reactor and ``treq`` testing
        infrastructure necessary to support asynchronous HTTP server endpoints.
        """
        d = ensureDeferred(d)
        result = []
        error = []
        d.addCallbacks(result.append, error.append)

        # Check for synchronous HTTP endpoint handler:
        if result:
            return result[0]
        if error:
            error[0].raiseException()

        # OK, no result yet, probably async HTTP endpoint handler, so advance
        # time, flush treq, and try again:
        for i in range(10_000):
            self.clock.advance(0.001)
            self.treq.flush()
            if result:
                break
            # By putting the sleep at the end, tests that are completely
            # synchronous and don't use threads will have already broken out of
            # the loop, and so will finish without any sleeps. This allows them
            # to run as quickly as possible.
            #
            # However, some tests do talk to APIs that use a thread pool on the
            # backend, so we need to allow actual time to pass for those.
            time.sleep(0.001)

        if result:
            return result[0]
        if error:
            error[0].raiseException()

        raise RuntimeError(
            "We expected given Deferred to have result already, but it wasn't. "
            + "This is probably a test design issue."
        )

class StorageClientWithHeadersOverride(object):
    """Wrap ``StorageClient`` and override sent headers."""

    def __init__(self, storage_client, add_headers):
        self.storage_client = storage_client
        self.add_headers = add_headers

    def __getattr__(self, attr):
        return getattr(self.storage_client, attr)

    def request(self, *args, headers=None, **kwargs):
        if headers is None:
            headers = Headers()
        for key, value in self.add_headers.items():
            headers.setRawHeaders(key, [value])
        return self.storage_client.request(*args, headers=headers, **kwargs)

@contextmanager
def assert_fails_with_http_code(test_case: SyncTestCase, code: int):
    """
    Context manager that asserts the code fails with the given HTTP response
    code.
    """
    with test_case.assertRaises(ClientException) as e:
        try:
            yield
        finally:
            pass
    test_case.assertEqual(e.exception.code, code)

class GenericHTTPAPITests(SyncTestCase):
    """
    Tests of HTTP client talking to the HTTP server, for generic HTTP API
    endpoints and concerns.
    """

    def setUp(self):
        super(GenericHTTPAPITests, self).setUp()
        disable_thread_pool_for_test(self)
        self.http = self.useFixture(HttpTestFixture())

    def test_missing_authentication(self) -> None:
        """
        If nothing is given in the ``Authorization`` header at all an
        ``Unauthorized`` response is returned.
        """
        client = StubTreq(self.http.http_server.get_resource())
        response = self.http.result_of_with_flush(
            client.request(
                "GET",
                "http://127.0.0.1/storage/v1/version",
            ),
        )
        self.assertThat(response.code, Equals(http.UNAUTHORIZED))

    def test_bad_authentication(self):
        """
        If the wrong swissnum is used, an ``Unauthorized`` response code is
        returned.
        """
        client = StorageClientGeneral(
            StorageClient(
                DecodedURL.from_text("http://127.0.0.1"),
                b"something wrong",
                treq=StubTreq(self.http.http_server.get_resource()),
                pool=None,
                clock=self.http.clock,
                analyze_response=response_is_not_html,
            )
        )
        with assert_fails_with_http_code(self, http.UNAUTHORIZED):
            self.http.result_of_with_flush(client.get_version())

    def test_unsupported_mime_type(self):
        """
        The client can request mime types other than CBOR, and if they are
        unsupported a NOT ACCEPTABLE (406) error will be returned.
        """
        client = StorageClientGeneral(
            StorageClientWithHeadersOverride(self.http.client, {"accept": "image/gif"})
        )
        with assert_fails_with_http_code(self, http.NOT_ACCEPTABLE):
            self.http.result_of_with_flush(client.get_version())

    def test_version(self):
        """
        The client can return the version.

        We ignore available disk space and max immutable share size, since that
        might change across calls.
        """
        client = StorageClientGeneral(self.http.client)
        version = self.http.result_of_with_flush(client.get_version())
        version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop(
            b"available-space"
        )
        version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop(
            b"maximum-immutable-share-size"
        )
        expected_version = self.http.storage_server.get_version()
        expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop(
            b"available-space"
        )
        expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop(
            b"maximum-immutable-share-size"
        )
        self.assertEqual(version, expected_version)

    def test_server_side_schema_validation(self):
        """
        Ensure that schema validation is happening: invalid CBOR should result
        in bad request response code (error 400).

        We don't bother checking every single request, the API on the
        server-side is designed to require a schema, so it validates
        everywhere.  But we check at least one to ensure we get correct
        response code on bad input, so we know validation happened.
        """
        upload_secret = urandom(32)
        lease_secret = urandom(32)
        storage_index = urandom(16)
        url = self.http.client.relative_url(
            "/storage/v1/immutable/" + _encode_si(storage_index)
        )
        message = {"bad-message": "missing expected keys"}

        response = self.http.result_of_with_flush(
            self.http.client.request(
                "POST",
                url,
                lease_renew_secret=lease_secret,
                lease_cancel_secret=lease_secret,
                upload_secret=upload_secret,
                message_to_serialize=message,
            )
        )
        self.assertEqual(response.code, http.BAD_REQUEST)

class ImmutableHTTPAPITests(SyncTestCase):
    """
    Tests for immutable upload/download APIs.
    """

    def setUp(self):
        super(ImmutableHTTPAPITests, self).setUp()
        disable_thread_pool_for_test(self)
        self.http = self.useFixture(HttpTestFixture())
        self.imm_client = StorageClientImmutables(self.http.client)
        self.general_client = StorageClientGeneral(self.http.client)

    def create_upload(self, share_numbers, length):
        """
        Create a write bucket on server, return:

            (upload_secret, lease_secret, storage_index, result)
        """
        upload_secret = urandom(32)
        lease_secret = urandom(32)
        storage_index = urandom(16)
        created = self.http.result_of_with_flush(
            self.imm_client.create(
                storage_index,
                share_numbers,
                length,
                upload_secret,
                lease_secret,
                lease_secret,
            )
        )
        return (upload_secret, lease_secret, storage_index, created)

    def test_upload_can_be_downloaded(self):
        """
        A single share can be uploaded in (possibly overlapping) chunks, and
        then a random chunk can be downloaded, and it will match the original
        file.

        We don't exercise the full variation of overlapping chunks because
        that's already done in test_storage.py.
        """
        length = 100
        expected_data = bytes(range(100))

        # Create a upload:
        (upload_secret, _, storage_index, created) = self.create_upload({1}, 100)
        self.assertEqual(
            created, ImmutableCreateResult(already_have=set(), allocated={1})
        )

        remaining = RangeMap()
        remaining.set(True, 0, 100)

        # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes.
        def write(offset, length):
            remaining.empty(offset, offset + length)
            return self.imm_client.write_share_chunk(
                storage_index,
                1,
                upload_secret,
                offset,
                expected_data[offset : offset + length],
            )

        upload_progress = self.http.result_of_with_flush(write(10, 10))
        self.assertEqual(
            upload_progress, UploadProgress(finished=False, required=remaining)
        )
        upload_progress = self.http.result_of_with_flush(write(30, 10))
        self.assertEqual(
            upload_progress, UploadProgress(finished=False, required=remaining)
        )
        upload_progress = self.http.result_of_with_flush(write(50, 10))
        self.assertEqual(
            upload_progress, UploadProgress(finished=False, required=remaining)
        )

        # Then, an overlapping write with matching data (15-35):
        upload_progress = self.http.result_of_with_flush(write(15, 20))
        self.assertEqual(
            upload_progress, UploadProgress(finished=False, required=remaining)
        )

        # Now fill in the holes:
        upload_progress = self.http.result_of_with_flush(write(0, 10))
        self.assertEqual(
            upload_progress, UploadProgress(finished=False, required=remaining)
        )
        upload_progress = self.http.result_of_with_flush(write(40, 10))
        self.assertEqual(
            upload_progress, UploadProgress(finished=False, required=remaining)
        )
        upload_progress = self.http.result_of_with_flush(write(60, 40))
        self.assertEqual(
            upload_progress, UploadProgress(finished=True, required=RangeMap())
        )

        # We can now read:
        for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]:
            downloaded = self.http.result_of_with_flush(
                self.imm_client.read_share_chunk(storage_index, 1, offset, length)
            )
            self.assertEqual(downloaded, expected_data[offset : offset + length])

    def test_write_with_wrong_upload_key(self):
        """
        A write with an upload key that is different than the original upload
        key will fail.
        """
        (upload_secret, _, storage_index, _) = self.create_upload({1}, 100)
        with assert_fails_with_http_code(self, http.UNAUTHORIZED):
            self.http.result_of_with_flush(
                self.imm_client.write_share_chunk(
                    storage_index,
                    1,
                    upload_secret + b"X",
                    0,
                    b"123",
                )
            )

    def test_allocate_buckets_second_time_different_shares(self):
        """
        If allocate buckets endpoint is called second time with different
        upload key on potentially different shares, that creates the buckets on
        those shares that are different.
        """
        # Create a upload:
        (upload_secret, lease_secret, storage_index, created) = self.create_upload(
            {1, 2, 3}, 100
        )

        # Write half of share 1
        self.http.result_of_with_flush(
            self.imm_client.write_share_chunk(
                storage_index,
                1,
                upload_secret,
                0,
                b"a" * 50,
            )
        )

        # Add same shares with a different upload key share 1 overlaps with
        # existing shares, this call shouldn't overwrite the existing
        # work-in-progress.
        upload_secret2 = b"x" * 2
        created2 = self.http.result_of_with_flush(
            self.imm_client.create(
                storage_index,
                {1, 4, 6},
                100,
                upload_secret2,
                lease_secret,
                lease_secret,
            )
        )
        self.assertEqual(created2.allocated, {4, 6})

        # Write second half of share 1
        self.assertTrue(
            self.http.result_of_with_flush(
                self.imm_client.write_share_chunk(
                    storage_index,
                    1,
                    upload_secret,
                    50,
                    b"b" * 50,
                )
            ).finished
        )

        # The upload of share 1 succeeded, demonstrating that second create()
        # call didn't overwrite work-in-progress.
        downloaded = self.http.result_of_with_flush(
            self.imm_client.read_share_chunk(storage_index, 1, 0, 100)
        )
        self.assertEqual(downloaded, b"a" * 50 + b"b" * 50)

        # We can successfully upload the shares created with the second upload secret.
        self.assertTrue(
            self.http.result_of_with_flush(
                self.imm_client.write_share_chunk(
                    storage_index,
                    4,
                    upload_secret2,
                    0,
                    b"x" * 100,
                )
            ).finished
        )

    def test_list_shares(self):
        """
        Once a share is finished uploading, it's possible to list it.
        """
        (upload_secret, _, storage_index, created) = self.create_upload({1, 2, 3}, 10)

        # Initially there are no shares:
        self.assertEqual(
            self.http.result_of_with_flush(self.imm_client.list_shares(storage_index)),
            set(),
        )

        # Upload shares 1 and 3:
        for share_number in [1, 3]:
            progress = self.http.result_of_with_flush(
                self.imm_client.write_share_chunk(
                    storage_index,
                    share_number,
                    upload_secret,
                    0,
                    b"0123456789",
                )
            )
            self.assertTrue(progress.finished)

        # Now shares 1 and 3 exist:
        self.assertEqual(
            self.http.result_of_with_flush(self.imm_client.list_shares(storage_index)),
            {1, 3},
        )

    def test_upload_bad_content_range(self):
        """
        Malformed or invalid Content-Range headers to the immutable upload
        endpoint result in a 416 error.
        """
        (upload_secret, _, storage_index, created) = self.create_upload({1}, 10)

        def check_invalid(bad_content_range_value):
            client = StorageClientImmutables(
                StorageClientWithHeadersOverride(
                    self.http.client, {"content-range": bad_content_range_value}
                )
            )
            with assert_fails_with_http_code(
                self, http.REQUESTED_RANGE_NOT_SATISFIABLE
            ):
                self.http.result_of_with_flush(
                    client.write_share_chunk(
                        storage_index,
                        1,
                        upload_secret,
                        0,
                        b"0123456789",
                    )
                )

        check_invalid("not a valid content-range header at all")
        check_invalid("bytes -1-9/10")
        check_invalid("bytes 0--9/10")
        check_invalid("teapots 0-9/10")

    def test_list_shares_unknown_storage_index(self):
        """
        Listing unknown storage index's shares results in empty list of shares.
        """
        storage_index = bytes(range(16))
        self.assertEqual(
            self.http.result_of_with_flush(self.imm_client.list_shares(storage_index)),
            set(),
        )

    def test_upload_non_existent_storage_index(self):
        """
        Uploading to a non-existent storage index or share number results in
        404.
        """
        (upload_secret, _, storage_index, _) = self.create_upload({1}, 10)

        def unknown_check(storage_index, share_number):
            with assert_fails_with_http_code(self, http.NOT_FOUND):
                self.http.result_of_with_flush(
                    self.imm_client.write_share_chunk(
                        storage_index,
                        share_number,
                        upload_secret,
                        0,
                        b"0123456789",
                    )
                )

        # Wrong share number:
        unknown_check(storage_index, 7)
        # Wrong storage index:
        unknown_check(b"X" * 16, 7)

    def test_multiple_shares_uploaded_to_different_place(self):
        """
        If a storage index has multiple shares, uploads to different shares are
        stored separately and can be downloaded separately.
        """
        (upload_secret, _, storage_index, _) = self.create_upload({1, 2}, 10)
        self.http.result_of_with_flush(
            self.imm_client.write_share_chunk(
                storage_index,
                1,
                upload_secret,
                0,
                b"1" * 10,
            )
        )
        self.http.result_of_with_flush(
            self.imm_client.write_share_chunk(
                storage_index,
                2,
                upload_secret,
                0,
                b"2" * 10,
            )
        )
        self.assertEqual(
            self.http.result_of_with_flush(
                self.imm_client.read_share_chunk(storage_index, 1, 0, 10)
            ),
            b"1" * 10,
        )
        self.assertEqual(
            self.http.result_of_with_flush(
                self.imm_client.read_share_chunk(storage_index, 2, 0, 10)
            ),
            b"2" * 10,
        )

    def test_mismatching_upload_fails(self):
        """
        If an uploaded chunk conflicts with an already uploaded chunk, a
        CONFLICT error is returned.
        """
        (upload_secret, _, storage_index, created) = self.create_upload({1}, 100)

        # Write:
        self.http.result_of_with_flush(
            self.imm_client.write_share_chunk(
                storage_index,
                1,
                upload_secret,
                0,
                b"0" * 10,
            )
        )

        # Conflicting write:
        with assert_fails_with_http_code(self, http.CONFLICT):
            self.http.result_of_with_flush(
                self.imm_client.write_share_chunk(
                    storage_index,
                    1,
                    upload_secret,
                    0,
                    b"0123456789",
                )
            )

    def test_timed_out_upload_allows_reupload(self):
        """
        If an in-progress upload times out, it is cancelled altogether,
        allowing a new upload to occur.
        """
        self._test_abort_or_timed_out_upload_to_existing_storage_index(
            lambda **kwargs: self.http.clock.advance(30 * 60 + 1)
        )

    def test_abort_upload_allows_reupload(self):
        """
        If an in-progress upload is aborted, it is cancelled altogether,
        allowing a new upload to occur.
        """

        def abort(storage_index, share_number, upload_secret):
            return self.http.result_of_with_flush(
                self.imm_client.abort_upload(storage_index, share_number, upload_secret)
            )

        self._test_abort_or_timed_out_upload_to_existing_storage_index(abort)

    def _test_abort_or_timed_out_upload_to_existing_storage_index(self, cancel_upload):
        """Start uploading to an existing storage index that then times out or aborts.

        Re-uploading should work.
        """
        # Start an upload:
        (upload_secret, _, storage_index, _) = self.create_upload({1}, 100)
        self.http.result_of_with_flush(
            self.imm_client.write_share_chunk(
                storage_index,
                1,
                upload_secret,
                0,
                b"123",
            )
        )

        # Now, the upload is cancelled somehow:
        cancel_upload(
            storage_index=storage_index, upload_secret=upload_secret, share_number=1
        )

        # Now we can create a new share with the same storage index without
        # complaint:
        upload_secret = urandom(32)
        lease_secret = urandom(32)
        created = self.http.result_of_with_flush(
            self.imm_client.create(
                storage_index,
                {1},
                100,
                upload_secret,
                lease_secret,
                lease_secret,
            )
        )
        self.assertEqual(created.allocated, {1})

        # And write to it, too:
        self.http.result_of_with_flush(
            self.imm_client.write_share_chunk(
                storage_index,
                1,
                upload_secret,
                0,
                b"ABC",
            )
        )

    def test_unknown_aborts(self):
        """
        Aborting uploads with an unknown storage index or share number will
        result 404 HTTP response code.
        """
        (upload_secret, _, storage_index, _) = self.create_upload({1}, 100)

        for si, num in [(storage_index, 3), (b"x" * 16, 1)]:
            with assert_fails_with_http_code(self, http.NOT_FOUND):
                self.http.result_of_with_flush(
                    self.imm_client.abort_upload(si, num, upload_secret)
                )

    def test_unauthorized_abort(self):
        """
        An abort with the wrong key will return an unauthorized error, and will
        not abort the upload.
        """
        (upload_secret, _, storage_index, _) = self.create_upload({1}, 100)

        # Failed to abort becaues wrong upload secret:
        with assert_fails_with_http_code(self, http.UNAUTHORIZED):
            self.http.result_of_with_flush(
                self.imm_client.abort_upload(storage_index, 1, upload_secret + b"X")
            )

        # We can still write to it:
        self.http.result_of_with_flush(
            self.imm_client.write_share_chunk(
                storage_index,
                1,
                upload_secret,
                0,
                b"ABC",
            )
        )

    def test_too_late_abort(self):
        """
        An abort of an already-fully-uploaded immutable will result in 405
        error and will not affect the immutable.
        """
        uploaded_data = b"123"
        (upload_secret, _, storage_index, _) = self.create_upload({0}, 3)
        self.http.result_of_with_flush(
            self.imm_client.write_share_chunk(
                storage_index,
                0,
                upload_secret,
                0,
                uploaded_data,
            )
        )

        # Can't abort, we finished upload:
        with assert_fails_with_http_code(self, http.NOT_ALLOWED):
            self.http.result_of_with_flush(
                self.imm_client.abort_upload(storage_index, 0, upload_secret)
            )

        # Abort didn't prevent reading:
        self.assertEqual(
            uploaded_data,
            self.http.result_of_with_flush(
                self.imm_client.read_share_chunk(
                    storage_index,
                    0,
                    0,
                    3,
                )
            ),
        )

    def test_lease_on_unknown_storage_index(self):
        """
        An attempt to renew an unknown storage index will result in a HTTP 404.
        """
        storage_index = urandom(16)
        secret = b"A" * 32
        with assert_fails_with_http_code(self, http.NOT_FOUND):
            self.http.result_of_with_flush(
                self.general_client.add_or_renew_lease(storage_index, secret, secret)
            )

class MutableHTTPAPIsTests(SyncTestCase):
    """Tests for mutable APIs."""

    def setUp(self):
        super(MutableHTTPAPIsTests, self).setUp()
        disable_thread_pool_for_test(self)
        self.http = self.useFixture(HttpTestFixture())
        self.mut_client = StorageClientMutables(self.http.client)

    def create_upload(self, data=b"abcdef"):
        """
        Utility that creates shares 0 and 1 with bodies
        ``{data}-{share_number}``.
        """
        write_secret = urandom(32)
        lease_secret = urandom(32)
        storage_index = urandom(16)
        self.http.result_of_with_flush(
            self.mut_client.read_test_write_chunks(
                storage_index,
                write_secret,
                lease_secret,
                lease_secret,
                {
                    0: TestWriteVectors(
                        write_vectors=[WriteVector(offset=0, data=data + b"-0")]
                    ),
                    1: TestWriteVectors(
                        write_vectors=[
                            WriteVector(offset=0, data=data),
                            WriteVector(offset=len(data), data=b"-1"),
                        ]
                    ),
                },
                [],
            )
        )
        return storage_index, write_secret, lease_secret

    def test_write_can_be_read_small_data(self):
        """
        Small written data can be read using ``read_share_chunk``.
        """
        self.write_can_be_read(b"abcdef")

    def test_write_can_be_read_large_data(self):
        """
        Large written data (50MB) can be read using ``read_share_chunk``.
        """
        self.write_can_be_read(b"abcdefghij" * 5 * 1024 * 1024)

    def write_can_be_read(self, data):
        """
        Written data can be read using ``read_share_chunk``.
        """
        lease_secret = urandom(32)
        storage_index = urandom(16)
        self.http.result_of_with_flush(
            self.mut_client.read_test_write_chunks(
                storage_index,
                urandom(32),
                lease_secret,
                lease_secret,
                {
                    0: TestWriteVectors(
                        write_vectors=[WriteVector(offset=0, data=data)]
                    ),
                },
                [],
            )
        )
        read_data = self.http.result_of_with_flush(
            self.mut_client.read_share_chunk(storage_index, 0, 0, len(data))
        )
        self.assertEqual(read_data, data)

    def test_read_before_write(self):
        """In combo read/test/write operation, reads happen before writes."""
        storage_index, write_secret, lease_secret = self.create_upload()
        result = self.http.result_of_with_flush(
            self.mut_client.read_test_write_chunks(
                storage_index,
                write_secret,
                lease_secret,
                lease_secret,
                {
                    0: TestWriteVectors(
                        write_vectors=[WriteVector(offset=1, data=b"XYZ")]
                    ),
                },
                [ReadVector(0, 8)],
            )
        )
        # Reads are from before the write:
        self.assertEqual(
            result,
            ReadTestWriteResult(
                success=True, reads={0: [b"abcdef-0"], 1: [b"abcdef-1"]}
            ),
        )
        # But the write did happen:
        data0 = self.http.result_of_with_flush(
            self.mut_client.read_share_chunk(storage_index, 0, 0, 8)
        )
        data1 = self.http.result_of_with_flush(
            self.mut_client.read_share_chunk(storage_index, 1, 0, 8)
        )
        self.assertEqual((data0, data1), (b"aXYZef-0", b"abcdef-1"))

    def test_conditional_write(self):
        """Uploads only happen if the test passes."""
        storage_index, write_secret, lease_secret = self.create_upload()
        result_failed = self.http.result_of_with_flush(
            self.mut_client.read_test_write_chunks(
                storage_index,
                write_secret,
                lease_secret,
                lease_secret,
                {
                    0: TestWriteVectors(
                        test_vectors=[TestVector(1, 4, b"FAIL")],
                        write_vectors=[WriteVector(offset=1, data=b"XYZ")],
                    ),
                },
                [],
            )
        )
        self.assertFalse(result_failed.success)

        # This time the test matches:
        result = self.http.result_of_with_flush(
            self.mut_client.read_test_write_chunks(
                storage_index,
                write_secret,
                lease_secret,
                lease_secret,
                {
                    0: TestWriteVectors(
                        test_vectors=[TestVector(1, 4, b"bcde")],
                        write_vectors=[WriteVector(offset=1, data=b"XYZ")],
                    ),
                },
                [ReadVector(0, 8)],
            )
        )
        self.assertTrue(result.success)
        self.assertEqual(
            self.http.result_of_with_flush(
                self.mut_client.read_share_chunk(storage_index, 0, 0, 8)
            ),
            b"aXYZef-0",
        )

    def test_list_shares(self):
        """``list_shares()`` returns the shares for a given storage index."""
        storage_index, _, _ = self.create_upload()
        self.assertEqual(
            self.http.result_of_with_flush(self.mut_client.list_shares(storage_index)),
            {0, 1},
        )

    def test_non_existent_list_shares(self):
        """A non-existent storage index errors when shares are listed."""
        with self.assertRaises(ClientException) as exc:
            self.http.result_of_with_flush(self.mut_client.list_shares(urandom(32)))
        self.assertEqual(exc.exception.code, http.NOT_FOUND)

    def test_wrong_write_enabler(self):
        """Writes with the wrong write enabler fail, and are not processed."""
        storage_index, write_secret, lease_secret = self.create_upload()
        with self.assertRaises(ClientException) as exc:
            self.http.result_of_with_flush(
                self.mut_client.read_test_write_chunks(
                    storage_index,
                    urandom(32),
                    lease_secret,
                    lease_secret,
                    {
                        0: TestWriteVectors(
                            write_vectors=[WriteVector(offset=1, data=b"XYZ")]
                        ),
                    },
                    [ReadVector(0, 8)],
                )
            )
        self.assertEqual(exc.exception.code, http.UNAUTHORIZED)

        # The write did not happen:
        self.assertEqual(
            self.http.result_of_with_flush(
                self.mut_client.read_share_chunk(storage_index, 0, 0, 8)
            ),
            b"abcdef-0",
        )

class SharedImmutableMutableTestsMixin:
    """
    Shared tests for mutables and immutables where the API is the same.
    """

    KIND: str  # either "mutable" or "immutable"
    general_client: StorageClientGeneral
    client: Union[StorageClientImmutables, StorageClientMutables]
    clientFactory: Callable[
        [StorageClient], Union[StorageClientImmutables, StorageClientMutables]
    ]

    def upload(self, share_number: int, data_length=26) -> Tuple[bytes, bytes, bytes]:
        """
        Create a share, return (storage_index, uploaded_data, lease secret).
        """
        raise NotImplementedError

    def get_leases(self, storage_index: bytes) -> Iterable[LeaseInfo]:
        """Get leases for the storage index."""
        raise NotImplementedError()

    def test_advise_corrupt_share(self):
        """
        Advising share was corrupted succeeds from HTTP client's perspective,
        and calls appropriate method on server.
        """
        corrupted = []
        self.http.storage_server.advise_corrupt_share = lambda *args: corrupted.append(
            args
        )

        storage_index, _, _ = self.upload(13)
        reason = "OHNO \u1235"
        self.http.result_of_with_flush(
            self.client.advise_corrupt_share(storage_index, 13, reason)
        )

        self.assertEqual(
            corrupted,
            [(self.KIND.encode("ascii"), storage_index, 13, reason.encode("utf-8"))],
        )

    def test_advise_corrupt_share_unknown(self):
        """
        Advising an unknown share was corrupted results in 404.
        """
        storage_index, _, _ = self.upload(13)
        reason = "OHNO \u1235"
        self.http.result_of_with_flush(
            self.client.advise_corrupt_share(storage_index, 13, reason)
        )

        for si, share_number in [(storage_index, 11), (urandom(16), 13)]:
            with assert_fails_with_http_code(self, http.NOT_FOUND):
                self.http.result_of_with_flush(
                    self.client.advise_corrupt_share(si, share_number, reason)
                )

    def test_lease_renew_and_add(self):
        """
        It's possible the renew the lease on an uploaded mutable/immutable, by
        using the same renewal secret, or add a new lease by choosing a
        different renewal secret.
        """
        # Create a storage index:
        storage_index, _, lease_secret = self.upload(0)

        [lease] = self.get_leases(storage_index)
        initial_expiration_time = lease.get_expiration_time()

        # Time passes:
        self.http.clock.advance(167)

        # We renew the lease:
        self.http.result_of_with_flush(
            self.general_client.add_or_renew_lease(
                storage_index, lease_secret, lease_secret
            )
        )

        # More time passes:
        self.http.clock.advance(10)

        # We create a new lease:
        lease_secret2 = urandom(32)
        self.http.result_of_with_flush(
            self.general_client.add_or_renew_lease(
                storage_index, lease_secret2, lease_secret2
            )
        )

        [lease1, lease2] = self.get_leases(storage_index)
        self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167)
        self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177)

    def test_read_of_wrong_storage_index_fails(self):
        """
        Reading from unknown storage index results in 404.
        """
        with assert_fails_with_http_code(self, http.NOT_FOUND):
            self.http.result_of_with_flush(
                self.client.read_share_chunk(
                    b"1" * 16,
                    1,
                    0,
                    10,
                )
            )

    def test_read_of_wrong_share_number_fails(self):
        """
        Reading from unknown storage index results in 404.
        """
        storage_index, _, _ = self.upload(1)
        with assert_fails_with_http_code(self, http.NOT_FOUND):
            self.http.result_of_with_flush(
                self.client.read_share_chunk(
                    storage_index,
                    7,  # different share number
                    0,
                    10,
                )
            )

    def test_read_with_negative_offset_fails(self):
        """
        Malformed or unsupported Range headers result in 416 (requested range
        not satisfiable) error.
        """
        storage_index, _, _ = self.upload(1)

        def check_bad_range(bad_range_value):
            client = self.clientFactory(
                StorageClientWithHeadersOverride(
                    self.http.client, {"range": bad_range_value}
                )
            )

            with assert_fails_with_http_code(
                self, http.REQUESTED_RANGE_NOT_SATISFIABLE
            ):
                self.http.result_of_with_flush(
                    client.read_share_chunk(
                        storage_index,
                        1,
                        0,
                        10,
                    )
                )

        # Bad unit
        check_bad_range("molluscs=0-9")
        # Negative offsets
        check_bad_range("bytes=-2-9")
        check_bad_range("bytes=0--10")
        # Negative offset no endpoint
        check_bad_range("bytes=-300-")
        check_bad_range("bytes=")
        # Multiple ranges are currently unsupported, even if they're
        # semantically valid under HTTP:
        check_bad_range("bytes=0-5, 6-7")
        # Ranges without an end are currently unsupported, even if they're
        # semantically valid under HTTP.
        check_bad_range("bytes=0-")

    def _read_with_no_range_test(self, data_length):
        """
        A read with no range returns the whole mutable/immutable.

        Actual test is defined in subclasses, to fix complaints from Hypothesis
        about the method having different executors.
        """
        storage_index, uploaded_data, _ = self.upload(1, data_length)
        response = self.http.result_of_with_flush(
            self.http.client.request(
                "GET",
                self.http.client.relative_url(
                    "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index))
                ),
            )
        )
        self.assertEqual(response.code, http.OK)
        self.assertEqual(
            self.http.result_of_with_flush(response.content()), uploaded_data
        )

    def test_validate_content_range_response_to_read(self):
        """
        The server responds to ranged reads with an appropriate Content-Range
        header.
        """
        storage_index, _, _ = self.upload(1, 26)

        def check_range(requested_range, expected_response):
            headers = Headers()
            headers.setRawHeaders("range", [requested_range])
            response = self.http.result_of_with_flush(
                self.http.client.request(
                    "GET",
                    self.http.client.relative_url(
                        "/storage/v1/{}/{}/1".format(
                            self.KIND, _encode_si(storage_index)
                        )
                    ),
                    headers=headers,
                )
            )
            self.assertEqual(
                response.headers.getRawHeaders("content-range"), [expected_response]
            )

        check_range("bytes=0-10", "bytes 0-10/*")
        check_range("bytes=3-17", "bytes 3-17/*")
        # TODO re-enable in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907
        # Can't go beyond the end of the mutable/immutable!
        # check_range("bytes=10-100", "bytes 10-25/*")

class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase):
    """Shared tests, running on immutables."""

    KIND = "immutable"
    clientFactory = StorageClientImmutables

    def setUp(self):
        super(ImmutableSharedTests, self).setUp()
        disable_thread_pool_for_test(self)
        self.http = self.useFixture(HttpTestFixture())
        self.client = self.clientFactory(self.http.client)
        self.general_client = StorageClientGeneral(self.http.client)

    def upload(self, share_number, data_length=26):
        """
        Create a share, return (storage_index, uploaded_data, lease_secret).
        """
        uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[
            :data_length
        ]
        upload_secret = urandom(32)
        lease_secret = urandom(32)
        storage_index = urandom(16)
        self.http.result_of_with_flush(
            self.client.create(
                storage_index,
                {share_number},
                data_length,
                upload_secret,
                lease_secret,
                lease_secret,
            )
        )
        self.http.result_of_with_flush(
            self.client.write_share_chunk(
                storage_index,
                share_number,
                upload_secret,
                0,
                uploaded_data,
            )
        )
        return storage_index, uploaded_data, lease_secret

    def get_leases(self, storage_index):
        return self.http.storage_server.get_leases(storage_index)

    @given(data_length=st.integers(min_value=1, max_value=300000))
    def test_read_with_no_range(self, data_length):
        """
        A read with no range returns the whole immutable.
        """
        return self._read_with_no_range_test(data_length)

class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase):
    """Shared tests, running on mutables."""

    KIND = "mutable"
    clientFactory = StorageClientMutables

    def setUp(self):
        super(MutableSharedTests, self).setUp()
        disable_thread_pool_for_test(self)
        self.http = self.useFixture(HttpTestFixture())
        self.client = self.clientFactory(self.http.client)
        self.general_client = StorageClientGeneral(self.http.client)

    def upload(self, share_number, data_length=26):
        """
        Create a share, return (storage_index, uploaded_data, lease_secret).
        """
        data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[:data_length]
        write_secret = urandom(32)
        lease_secret = urandom(32)
        storage_index = urandom(16)
        self.http.result_of_with_flush(
            self.client.read_test_write_chunks(
                storage_index,
                write_secret,
                lease_secret,
                lease_secret,
                {
                    share_number: TestWriteVectors(
                        write_vectors=[WriteVector(offset=0, data=data)]
                    ),
                },
                [],
            )
        )
        return storage_index, data, lease_secret

    def get_leases(self, storage_index):
        return self.http.storage_server.get_slot_leases(storage_index)

    @given(data_length=st.integers(min_value=1, max_value=300000))
    def test_read_with_no_range(self, data_length):
        """
        A read with no range returns the whole mutable.
        """
        return self._read_with_no_range_test(data_length)


from utils.utils import myprint as print

from utils.net.utils import Trainable, to_tensor, device
from utils.net.linear.mlp import MLP
from torch import nn, optim
from torch.nn import functional as F

class Classifier(Trainable):
    def __init__(self, input_size, classes, layers, hid_size=1000, hid_smooth=None, drop=0,
                 batch_norm=True, nl='leakyrelu', gated=False, excitability=False, excit_buffer=False, name=None):

        super().__init__()

        self.name = self.__class__.__name__ if name is None else name

        layers = int(layers)
        if layers < 1:
            raise ValueError("The classifier needs to have at least 1 fully-connected layer.")

        ######------SPECIFY MODEL------######

        self.mlp = MLP(input_size=input_size, output_size=classes, layers=layers, hid_size=hid_size,
                       hid_smooth=hid_smooth, batch_norm=batch_norm, gated=gated, excitability=excitability,
                       excit_buffer=excit_buffer, drop=drop, nl=nl, output='excitable')

        self.loss = nn.CrossEntropyLoss(reduction='mean')

    def forward(self, x):
        return self.mlp(x)

    def predict(self, x):
        mode = self.training
        self.eval()
        x = to_tensor(x).to(device)
        y_hat = self(x)
        predicted = y_hat.detach().argmax(1)
        self.train(mode=mode)
        return predicted.cpu().detach().numpy()

    def dist_loss(self, scores, target_scores, T=2.):
        """Compute knowledge-distillation (KD) loss given [scores] and [target_scores].

        Both [scores] and [target_scores] should be tensors, although [target_scores] should be repackaged.
        'Hyperparameter': temperature"""

        log_scores_norm = F.log_softmax(scores / T, dim=1)
        targets_norm = F.softmax(target_scores / T, dim=1)

        # Calculate distillation loss (see e.g., Li and Hoiem, 2017)
        KD_loss_unnorm = -(targets_norm * log_scores_norm)
        KD_loss_unnorm = KD_loss_unnorm.sum(dim=1)                      #--> sum over classes
        KD_loss_unnorm = KD_loss_unnorm.mean()                          #--> average over batch

        # normalize
        KD_loss = KD_loss_unnorm * T**2

        return KD_loss

    def train_batch(self, x, y, gen_x, gen_y, scores, gen_scores, rnt):
        super().train_batch()

        y_hat = self(x)
        total_loss = self.loss(y_hat, y)

        predicted = y_hat.detach().argmax(1)
        correct = (predicted == y).sum().item()

        if gen_x is not None:
            y_hat = self(gen_x)
            replay_loss = self.dist_loss(y_hat, gen_scores, T=2.0)
            total_loss = rnt * total_loss + (1 - rnt) * replay_loss

        total_loss.backward()
        self.optimizer.step()

        return total_loss.item(), correct


import subprocess
import clingo_interface

# Split target string into pos_target_list, neg_target_list
def split_target(target_str):
    pos_target_list, neg_target_list = [], []
    target_list = target_str.strip().split(".")
    for target in target_list:
        target = target.strip()
        if len(target) == 0:
            continue
        if target.find("-") == -1:
            pos_target_list.append(target)
        else:
            neg_target_list.append(target.replace('-', ''))
    return pos_target_list, neg_target_list

# Build an ILASP sample string
def build_ilasp_one_sample(idx, confidence, target_str, context_str, z_labels, preds, prefix):
    pos_target_list, neg_target_list = split_target(target_str)
    pos_target_str = ",".join(pos_target_list)
    neg_targt_str = ",".join(neg_target_list)
    logic_label_str = clingo_interface.labels2program(z_labels, preds, prefix)
    sample_str = '''
#pos(e%d@%d,{
    %s
},{
    %s
},{
    %s
    %s
}).
    ''' % (idx, confidence, pos_target_str, neg_targt_str, context_str, logic_label_str)
    return sample_str

# Build ILASP samples string
def build_ilasp_samples(confidence_list, target_str_list, context_str_list, z_labels_list, preds, prefix="_u"):
    samples_str = ""
    for i in range(len(target_str_list)):
        confidence = confidence_list[i]
        target_str = target_str_list[i]
        context_str = context_str_list[i]
        z_labels = z_labels_list[i]
        samples_str += build_ilasp_one_sample(i, confidence=confidence, target_str=target_str, context_str=context_str, z_labels=z_labels, preds=preds, prefix=prefix) + '\n'
    return samples_str

# Get the program learned by ILASP
def get_ilasp_result(cmd_list, inv_pred="", mode_file="", org_target_pred_name="invented_predicate", timeout=1200):
    try:
        result = subprocess.run(cmd_list, stdout=subprocess.PIPE, timeout=timeout)
        learned_program = result.stdout.decode()
        learned_program = learned_program.replace(";", ",")
    except subprocess.TimeoutExpired:
        learned_program = ""
        print("ILASP learning timeout!")
    # if len(inv_pred) > 0:
    #     learned_program += "\n" + "#defined %s/1."%(inv_pred)
    if len(mode_file) > 0:
        with open(mode_file,'r') as f:
            for line in f.readlines():
                line = line.strip()
                if len(line) > 0 and line.find("#mode") == -1 and line.find("~") == -1:
                    learned_program += "\n" + line.replace(org_target_pred_name, inv_pred)
    return learned_program

# Read background knowledge file and exclude some lines
def read_bk_file(bk_file, exclude_pred_list = []):
    bk_str = ""
    with open(bk_file,'r') as f:
        for line in f.readlines():
            flag = True
            for exclude_pred in exclude_pred_list:
                if exclude_pred in line:
                    flag = False
                    break
            if flag:
                bk_str += line
    return bk_str

# Read mode declaration file and exclude some lines
def read_mode_file(mode_file, org_target_pred_name, inv_pred, exclude_pred_list = []):
    mode_str = ""
    with open(mode_file,'r') as f:
        for line in f.readlines():
            flag = True
            for exclude_pred in exclude_pred_list:
                if exclude_pred in line or "#defined" in line:
                    flag = False
                    break
            if flag:
                mode_str += line.replace(org_target_pred_name, inv_pred)
    return mode_str

# Build the final ILASP program
def build_ilasp_program(kb_str, mode_file, inv_pred, confidence_list, target_str_list, context_str_list, z_labels_list, preds, exclude_pred_list = [], prefix="_u", max_penalty=None, maxv=None, out_file=""):
    org_target_pred_name = "invented_predicate"
    program_str = ""
    if max_penalty is not None:
        program_str += "#max_penalty(%d).\n"%(max_penalty)
    if maxv is not None:
        program_str += "#maxv(%d).\n"%(maxv)
    program_str += kb_str
    program_str += build_ilasp_samples(confidence_list=confidence_list, target_str_list=target_str_list, context_str_list=context_str_list, z_labels_list=z_labels_list, preds=preds, prefix="_u")
    program_str += read_mode_file(mode_file=mode_file, org_target_pred_name=org_target_pred_name, inv_pred=inv_pred, exclude_pred_list = exclude_pred_list)
    if len(out_file) > 0:
        with open(out_file, 'w') as f:
            f.write(program_str)
    return program_str

# Convert to one-vs-rest
def convert_ilasp_target(target_list, pos_pred):
    ilasp_target_list =  []
    for target in target_list:
        end = target.find("(")
        if target[:end]==pos_pred:
            ilasp_target_list.append(target)
        else:
            ilasp_target_list.append('-'+pos_pred+target[end:])
    return ilasp_target_list

# Contain the definition of preds
def contain_pred_def(line, preds):
    line = line.strip()
    for pred in preds:
        if line.startswith(pred):
            return True
    return False

# Conflict resolution for a kb line and learned program
def conflict_resol_line(kb_line, learned_program, new_pred):
    ret_list = []
    for learned_line in learned_program.split("\n"):
        if contain_pred_def(learned_line, [new_pred]):
            ret_list.append(conflict_resol_one(kb_line, learned_line))
    ret_line = "\n".join(ret_list)
    return ret_line

def gen_subs(learned_line, kb_line):
    learned_para = learned_line[learned_line.find("(")+1:learned_line.find(")")].replace(" ","").split(",")
    kb_para = kb_line[kb_line.find("(")+1:kb_line.find(")")].replace(" ","").split(",")
    mapping = list(zip(learned_para, kb_para))
    return mapping

# Conflict resolution for a kb line and learned line
# Assume they are conflicting, have not check sat
def conflict_resol_one(kb_line, learned_line):
    ret_list = []
    start_idx = learned_line.find(":-")
    learned_body_literals = learned_line[start_idx+2:].replace(".","").split(",")
    learned2kb_var = gen_subs(learned_line, kb_line)
    for literal in learned_body_literals:
        literal = literal.strip()
        for (v1,v2) in learned2kb_var:
            literal = literal.replace(v1,v2)
        new_kb_line = kb_line.replace(".",",")+" not_"+literal+"."
        ret_list.append(new_kb_line)
    ret_line = "\n".join(ret_list)
    return ret_line

# Conflict resolution for a kb and learned program
def conflict_resolution(kb_str, learned_program, known_preds, new_pred):
    new_kb_str = ""
    for kb_line in kb_str.split("\n"):
        if contain_pred_def(kb_line, known_preds):
            print("May conflict with kb, trying to refine:\n", kb_line)
            refined_kb_line = conflict_resol_line(kb_line, learned_program, new_pred)
            print("Refined kb rule:\n", refined_kb_line)
            new_kb_str += refined_kb_line + "\n"
        else:
            new_kb_str += kb_line + "\n"
    if new_kb_str.find('even_1(X,Y) :- div_2(X), div_2(Y), not_div_3(X).') and \
        new_kb_str.find('odd_1(X,Y) :- not_div_2(X), not_div_2(Y), not_div_3(X).'): # Found correct, add CWA
        new_kb_str += '''
-new_1(X,Y) :- misc(X,Y), not_div_3(X).
-new_1(X,Y) :- misc(X,Y), not_div_3(Y).
    '''
    return new_kb_str

if __name__ == "__main__":
    pass