# This is the quadplay server script. The quadplay server is a stock
# web server with the following extensions:
#
#  - POST commands take a JSON argument with a `command` that is:
#
#    "write_file":    Write the specified contents to disk at the specified location.
#                     Used by the IDE to save files.
#
#    "new_game":      Create a new game by copying and renaming the specified template
#                     and return the URL of the new game, or return an error if the
#                     directory already existed.
#
#    "open":          Open the specified file in finder/explorer or using an external
#                     application editor
#
#    "quit":          Shut down the server itself (only in --nativeapp mode)
#
#    "update":        Update quadplay via git pull or downloading from github, depending on how it was installed
#
#
#  - GET commands:
#
#    - Are limited to the quadplay directory, the directory passed on the
#      command line, and $HOME/my_game
#
#    - Special quad_filepath + '/console/' + ... URLs that on a
#      quadplay server are dynamically generated instead of files:
#
#       - `_assets.json` gives a listing of all assets in the
#         specified query directory and the quadplay built-in
#         directories.  This combines raw and json packaged assets.
#
#       - `_scripts.json` gives a listing of all .pyxl files in
#         the specified query directory and the quadplay built-in
#         scripts directory.
#
#       - `_docs.json` gives a listing of all .md, .html, and
#         .pdf files in the specified query directory.
#
#       - `games.json` gives a listing of all games in subdirectories
#         of examples/, games/, and my_quadplay/ (not recursive). This
#         is backed by an actual file when not using a quadplay
#         server.
#
#       - `_config.json` describes the available external
#         applications, which are a subset of
#         console/external-applications.json. Also includes
#         information on the git version when running from private
#         developer git.
#
#       - `_update_progress.json` reports on the status of the update
#         thread once launched.
#
# A "path" in this code is a file-system path, where '/' means the filesystem true root
#
# A "webpath" is a URL subpath, where '/' means the server root = filesystem root_path

import os, time, platform, sys, threading, socket, argparse, multiprocessing, json, ssl, codecs, glob, shutil, re, base64, random, getpass, inspect, subprocess, workjson, signal, shlex, tkinter, tkinter.messagebox, urllib, urllib.request

quadplay_origin = 'git clone' if os.path.isdir(os.path.join(os.path.dirname(__file__), '../.git')) else 'downloaded release'
if quadplay_origin == 'git clone' and os.path.exists(os.path.join(os.path.dirname(__file__), '../_dev')):
    quadplay_origin = 'development git clone'

# Catch a operator error with git. This binary file should be large if LFS is present:
if quadplay_origin == 'development git clone' and os.stat(os.path.join(os.path.dirname(__file__), '../console/xbox_controller.png')).st_size < 1000:
    print('You are running the development build of quadplay without git LFS, so the binary files are not present. Enable LFS and pull.')
    sys.exit(2)

# If needed, this is how we'd restrict the version. The current implementation does not need to restrict the version:
# if sys.version_info[0] < 3 or sys.version_info[1] < 7:
#     print('Sorry, you are using Python ' + sys.version + ' and quadplay requires Python 3.7 or newer. Download and install from https://python.org')
#     sys.exit(-1)

__doc__ = "quadplay console server"

python_version = str(sys.version_info.major) + '.' + str(sys.version_info.minor)

# Largest common prefix of quad_filepath and game_filepath. Root for web serving
server_root_filepath = None

# Path to the token.txt file
token_absfilepath = None

# Magic path that the browser will use to communicate that it wants an asset listing
# or game listing for import in IDE mode
asset_query_webpath = None
script_query_webpath = None
doc_query_webpath = None
game_query_webpath = None
config_query_webpath = None
update_progress_query_webpath = None

# Subdirectories of server_root_filepath that are permitted for http access.
# These are relative to server_root_filepath and contain a leading slash because
# they are url subpaths relative to quad://
webpath_allowlist = None

isWindows = (platform.system() == 'Windows')
isMacOS   = (platform.system() == 'Darwin')
isLinux   = (platform.system() == 'Linux')

# Authentication token, used to ensure that only browsers launched by this server can
# ask it to POST
token = None

def ignore(*args): pass

# Assigned in main based on the --quiet flag
maybe_print = ignore

##########################################################################
# Version detection

# Parses a version.js file with an embedded 'version = yyyy.mm.dd.hh'
# string and returns it as a single integer for version comparisons,
# as well as the human-readable version string.
def parse_version_js(file_string):
    version_parser = re.compile("^ *const *version *= *['\"]([.0-9]+)*['\"] *;")
    try:
        version_string = version_parser.search(file_string).group(1)

        year, month, day, hour = [int(x) for x in version_string.split('.')]

        # Months and years have varying length. That doesn't matter.  We
        # don't need a linear time number. We need one that monotonically
        # increases. Think of this as parsing a number with an irregular
        # base per digit.
        return {'value': ((year * 12 + month) * 32 + day) * 24 + hour,
                'text': version_string}
    except:

        return {'value': 0, 'text': file_string}

with open(os.path.join(os.path.dirname(__file__), '../console/version.js')) as file:
    installed_quadplay_version = parse_version_js(file.read())

##########################################################################

# Ensure that slashes and case are consistent when on Windows, and make absolute
def canonicalize_filepath(path): return os.path.normcase(os.path.abspath(path)).replace('\\', '/')

def fatal_error(message):
    # Initialize the GUI
    window_root = tkinter.Tk()
    window_root.withdraw()

    # Message Box
    print("Quadplay Error: " + message)
    tkinter.messagebox.showerror("Quadplay Error", message)
    sys.exit(-1)

def parse_args():
    """ parse arguments out of sys.argv """
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )

    parser.add_argument(
        '--serve',
        action='store_true',
        default=False,
        help='serving mode - allow other machines to connect to this one to develop on mobile devices'
    )

    parser.add_argument(
        '--nativeapp',
        action='store_true',
        default=False,
        help='Attempt to open quadplay as a native app window on platforms with Chrome or Edge. Defaults to True for the quadplay script and False for the quadplay-server script.'
    )

    parser.add_argument(
        '--quiet',
        action='store_true',
        default=False,
        help='Minimize console output, eliminating the console window itself if possible.'
    )

    parser.add_argument(
        '--my_quadplay',
        default=os.environ.get('MY_QUADPLAY', '~/my_quadplay'),
        help="Set the path to local games, which defaults to my_quadplay in the current user's home directory or the MY_QUADPLAY environment variable."
    )

    parser.add_argument(
        '--noupdatecheck',
        default=False,
        help="Do not check for newer versions of quadplay on github. Never checks when in kiosk mode")

    parser.add_argument(
        'gamepath',
        type=str,
        default='',
        nargs='?',
        help=(
            'Game to load.  If not specified, loads a default game.'
            ' Example: examples/accel_demo'
        )
    )

    parser.add_argument(
        '--kiosk',
        action='store_true',
        default=False,
        help='kiosk mode - launch full screen, without the IDE or other controls'
    )

    parser.add_argument(
        '--play',
        action='store_true',
        default=False,
        help='play mode - launch without the IDE'
    )

    parser.add_argument(
        '--offline',
        action='store_true',
        default=False,
        help='Prevent online multiplayer as both host and guest'
    )

    parser.add_argument(
        "-p",
        "--port",
        type=int,
        default=8000,
        help="Port to run the server on."
    )

    return parser.parse_args()

# Args must be initialized to a constant before launchServer is defined
# because of the way that multiprocess serialization works on ThreadingHTTPServer
args = parse_args()

# Where the user's games are stored. Will be made relative to the CWD later
my_games_filepath = canonicalize_filepath(os.path.expanduser(args.my_quadplay))

if my_games_filepath.find(" ") != -1:
    fatal_error("The my_quadplay directory (" + my_games_filepath + ") may not contain spaces in this release. Use the --my_quadplay command line argument to select a path without spaces in the name.")

if len(my_games_filepath) == 0 or (isWindows and (len(my_games_filepath) < 3 or my_games_filepath[1:3] != ':/')) or (not isWindows and (len(my_games_filepath) < 2 or my_games_filepath[0] != '/')):
    fatal_error("The my_quadplay directory (" + my_games_filepath + ") must be an absolute path.")

if not os.path.exists(my_games_filepath):
    if my_games_filepath == canonicalize_filepath(os.path.expanduser('~/my_quadplay')):
        # Create the games path and put a .gitignore in it if there isn't
        # one already
        os.makedirs(my_games_filepath)
        shutil.copyfile('.gitignore', os.path.join(my_games_filepath, '.gitignore'))
    else:
        fatal_error("The specified my_quadplay directory (" + my_games_filepath + ") could not be found.")

def remove_leading_slash(path): return path[1:] if len(path) > 0 and path[0] == '/' else path
def remove_trailing_slash(path): return path[:-1] if len(path) > 0 and path[-1] == '/' else path

try:
    # Python 3.7+
    from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
    QuadplayHTTPServer = ThreadingHTTPServer
    maybe_print('Initializing Quadplay server using Python ' + python_version + ' ThreadingHTTPServer')

except ImportError:
    try:
        # Python 3+
        from http.server import HTTPServer, SimpleHTTPRequestHandler
        maybe_print('Initializing Quadplay server using Python ' + python_version + ' HTTPServer')
        QuadplayHTTPServer = HTTPServer
    except ImportError:
        # Python 2
        from SimpleHTTPServer import SimpleHTTPRequestHandler
        import SocketServer
        QuadplayHTTPServer = SocketServer.TCPServer
        maybe_print('Initializing Quadplay server using Python ' + python_version + ' SocketServer.TCPServer')

# Returns a string of the line number that calls this function
def this_line_number(): return str(inspect.currentframe().f_back.f_lineno)

##############################################################################

# Throws an exception with the output on a non-zero error code,
# otherwise returns the command's output.
def run_shell_command(cmd):
    maybe_print(cmd)
    with os.popen(cmd) as stream:
        output = stream.read()
        if stream.close():
            raise Exception(output)
        else:
            maybe_print(output)
            return output

class UpdateThread(threading.Thread):

    def __init__(self):
        super(UpdateThread, self).__init__(name='update_thread', daemon=True)
        self._status = 'Running'
        self._status_lock = threading.Lock()

    def set_status(self, value):
        with self._status_lock:
            self._status = value

    def get_status(self):
        with self._status_lock:
            return self._status

    def run(self):
        try:
            if self.update():
                self.set_status('Done. Restart server.')
            else:
                self.set_status('Done. Restart client.')
        except Exception as e:
            maybe_print('Update failed: ' + str(e))
            self.set_status('Failed: ' + str(e))

    # Returns true if the server needs to be restarted.
    # Raises an exception if the update fails.
    def update(self):
        old_dir = os.getcwd()

        # go to the quadplay root dir
        os.chdir(os.path.join(os.path.dirname(__file__), '..'))

        # Track if this script itself changed
        my_previous_timestamp = os.path.getmtime(__file__)

        try:
            if quadplay_origin == 'git clone' or quadplay_origin == 'development git clone':
                # Verify that we have git
                if not shutil.which('git'):
                    raise Exception('Your quadplay was installed via git clone but the updater cannot find the git program to pull the latest version. Delete the .git directory or manually update.')

                # Run git
                run_shell_command('git pull')

            elif quadplay_origin == 'downloaded release':
                os.mkdir('temp')
                run_shell_command('curl --location --output temp/quadplay-install.zip https://github.com/morgan3d/quadplay/archive/refs/heads/main.zip')
                os.chdir('temp')
                try:
                    run_shell_command('tar -xf quadplay-install.zip')

                    # Copy files
                    shutil.copytree('quadplay-main', '..', dirs_exist_ok = True)
                    shutil.rmtree('temp')
                except:
                    # Try to clean up
                    os.chdir('..')
                    shutil.rmtree('temp')
                    raise

            # Success
            os.chdir(old_dir)
            return os.path.getmtime(__file__) != my_previous_timestamp

        except:
            os.chdir(old_dir)
            raise

# The global updating thread. Initialized when the client
# requests the update.
update_thread = None

##############################################################################
# Handles serving from multiple directories. Overrides the default
# restriction to the CWD with its own security allowlist. Instantiated
# per request.
#
# See https://github.com/python/cpython/blob/master/Lib/http/server.py for
# the base class implementation and internal methods
class QuadplayHTTPRequestHandler(SimpleHTTPRequestHandler):
    # Directories that should be excluded from filesystem searches
    special_dir_regex = re.compile('(^|/)(journal|screenshots|metadata|backlot|graveyard)/', re.IGNORECASE)

    # Files that can be written or deleted by the IDE
    mutable_file_regex = re.compile(r'\.(json|xml|pyxl|png|pdf|yml|yaml|html|txt|md|tmx|mp3)$')

    # Files that should not print error messages to the script's
    # terminal if not found (there is no way to stop these from
    # printing errors to the *browser's* console).
    silent_404_file_regex = re.compile(r'(\.debug\.json|(^|/)label(64|128)\.png|favicon\.png)$')

    # Add custom headers
    def end_headers(self):
        # CORS:
        self.send_header('Access-Control-Allow-Origin', '*')

        # Required to enable high precision timer in Firefox and Chrome:
        self.send_header('Cross-Origin-Embedder-Policy', 'require-corp')
        self.send_header('Cross-Origin-Opener-Policy', 'same-origin')

        # Try to reduce overcaching
        self.send_header('Cache-Control', 'must-revalidate')
        SimpleHTTPRequestHandler.end_headers(self)

    def list_directory(self, path):
        # Prevent directory listing
        self.send_error(404, 'No permission to list directory (' + this_line_number() + ')')
        return None

    # Returns False on security error
    def prepare_mutating_request(self):
        if args.serve:
           maybe_print('Refused POST/DELETE command while running with --serve')
           return False

        if self.client_address[0] != '127.0.0.1':
           maybe_print('Refused POST/DELETE command from non-local address', self.client_address[0])
           return False

        content_len = int(self.headers['Content-Length'])
        object = json.loads(self.rfile.read(content_len))

        request_token = object['token']
        if request_token != token:
            maybe_print('WARNING: Ignored request without proper token (expected ', token, '/ received ', request_token, ')')
            response = json.dumps('Bad token', separators = (',', ':'));
            self.send_response(401)
            self.send_header('Content-type', 'text/json')
            self.send_header('Content-length', len(response))
            self.end_headers()
            self.wfile.write(response.encode('utf8'))
            return False

        return object

    def do_DELETE(self):
        object = self.prepare_mutating_request()
        if not object: return
        path_parts = self.path.split('?')
        webpath = os.path.normpath(path_parts[0]).replace('\\', '/')

        maybe_print('DELETE', webpath)
        code = 200
        filename = remove_leading_slash(webpath)
        if not re.search(self.mutable_file_regex, filename) or not any([webpath.startswith(prefix) for prefix in webpath_allowlist]):
            maybe_print('WARNING: Ignored illegal request to delete file (' + this_line_number() + ')', filename)
            response_obj = 'Illegal'
            code = 401
        else:
            maybe_print('deleted file', filename)
            os.remove(filename)

        response = json.dumps('OK', separators = (',', ':'));
        self.send_response(code)
        self.send_header('Content-type', 'text/json')
        self.send_header('Content-length', len(response))
        self.end_headers()
        self.wfile.write(response.encode('utf8'))

    # Used for the IDE to write files
    def do_POST(self):
        global update_thread
        # maybe_print('\n\nReceived POST from ', self.client_address[0], '\n\n')

        object = self.prepare_mutating_request()
        if not object: return

        command = object['command']

        response_obj = None
        code = 201
        if command == 'write_file':
            webpath = object['url']
            filename = remove_leading_slash(webpath)
            encoding = object['encoding']
            contents = object['contents']

            if not re.search(self.mutable_file_regex, filename) or not any([webpath.startswith(prefix) for prefix in webpath_allowlist]):
                maybe_print('WARNING: Ignored illegal request to write file (' + this_line_number() + ')', filename)
                if not re.search(self.mutable_file_regex, filename): maybe_print('  Legal file regex is', self.mutable_file_regex)
                if not any([webpath.startswith(prefix) for prefix in webpath_allowlist]): maybe_print('  Web path allow list is', webpath_allowlist)
                response_obj = 'Illegal'
                code = 401
            else:
                if encoding == 'binary':
                    # Contents is base64 encoded for the JSON transmission.
                    # Convert back to bytes
                    contents = base64.standard_b64decode(contents)
                    with open(filename, 'wb') as f:
                        f.write(contents)
                        # See comment below about flushing
                        f.flush()
                        f.close()
                else:
                    with codecs.open(filename, 'w', encoding) as f:
                        f.write(contents)
                        # Documentation says that closing the file flushes it, however
                        # on Windows we get race conditions where new files do not exist
                        # sometimes for a subsequent read even though they have been
                        # closed. So, we flush explicitly here, which either affects
                        # the timing just enough to make it succeed, or hopefully actually
                        # blocks until the file is written.
                        f.flush()
                        f.close()
                maybe_print('Wrote', encoding, 'file', filename)

                if isWindows:
                    # Because of the windows failure to flush files,
                    # there can be a race condition in the case of
                    # file renaming, where the client assumes the write
                    # has finished and then tries to read the same file.
                    # So, check here:
                    max_iterations = 10
                    while not os.path.exists(filename) and max_iterations > 0:
                        time.sleep(0.02)
                        --max_iterations
                response_obj = 'OK'

        elif command == 'quit':
            # Used in nativeapp mode for the browser to tell the
            # (invisible) server to exit. Windows and Linux will
            # detect the browser closing, so this is only essential on
            # macOS where the menu option closes the TAB but leaves
            # the browser process still running.
            print('quadplay server terminated')

            # Hard exit, rather than raising an exception. Needed
            # because of threads
            if isWindows:
                os._exit(0)
            else:
                os.kill(os.getpid(), 9)

            response_obj = 'OK'

        elif command == 'open':
            app = object['app']
            filename = '"' + object['file'] + '"'
            cmd = ''
            if app == '<finder>':
                if isMacOS:
                    cmd = 'open -R "' + filename + '"'
                else:
                    cmd = 'start "" "' + os.path.dirname(filename) + '"'
            elif isMacOS:
                cmd = 'open -a "' + app + '" "' + filename + '"'

            os.system(cmd)
            response_obj = 'OK'

        elif command == 'new_game':
            dir_name = object['dir_name']
            game_name = object['game_name']
            src_url = object['src_url']

            dst_path = my_games_filepath + '/' + dir_name

            # Convert src_url to src_path + src_filename
            src_path = re.sub(r'^https?://[^/]*/', '', src_url)
            src_path = re.sub(r'^quad://', quad_filepath, src_path)
            src_path = remove_leading_slash(os.path.normpath(src_path).replace('\\', '/'))
            if src_path.endswith('.game.json'):
                src_path = src_path.split('/')
                src_filename = src_path[-1]
                src_path = '/'.join(src_path[:-1])
            else:
                # Remove trailing slash
                if src_path[-1] == '/': src_path[:-1]
                src_filename = src_path.split('/')[-1] + '.game.json'

            #src_path = os.path.join(quad_filepath, 'examples/starter')
            #src_filename = 'starter.game.json'

            maybe_print('Created game', dir_name, 'at', dst_path)

            # Copy and rename files
            if os.path.exists(dst_path):
                code = 406
                response_obj = {'message': dst_path + ' already exists'}
            else:
                # Recursively copy directory
                shutil.copytree(src_path, dst_path, ignore = shutil.ignore_patterns('.DS_Store', '*~', '#*'))
                #for filename in ['label64.png', 'label128.png', 'Play.pyxl']:
                #    shutil.copyfile(os.path.join(starter_path, filename), os.path.join(dst_path, filename))
                shutil.copyfile(os.path.join(quad_filepath, '.gitignore'), os.path.join(dst_path, '.gitignore'))

                # Parse the starter game
                game_json = workjson.load(os.path.join(src_path, src_filename))

                # Change the title
                game_json['title'] = game_name
                game_json['screenshot_tag'] = game_name

                # Save the new game
                with open(os.path.join(dst_path, dir_name + '.game.json'), 'wt', encoding='utf8') as f:
                    f.write(workjson.dumps(game_json, indent=4))

                response_obj = {'game': '/' + dst_path + '/'}

        elif command == 'update':
            maybe_print('Updating quadplay...')
            if not update_thread or (update_thread.get_status() != 'Running'):
                update_thread = UpdateThread()
                update_thread.start()
                response_obj = 'OK'
            else:
                maybe_print('Update already in progress.')
                response_obj = 'Update already in progress.'

        response = json.dumps(response_obj, separators = (',', ':'));
        self.send_response(code)
        self.send_header('Content-type', 'text/json')
        self.send_header('Content-length', len(response))
        self.end_headers()
        self.wfile.write(response.encode('utf8'))

    def do_GET(self):
        # Remove the query and collapse any '..' in the path
        path_parts = self.path.split('?')
        webpath = os.path.normpath(path_parts[0]).replace('\\', '/')
        query = path_parts[1] if len(path_parts) > 1 else ''

        #print('-----------------------------')
        #print('GET ' + path_parts[0])
        #print(' webpath = ' + webpath)

        # Security: Check if path has a prefix in webpath_allowlist
        if (not any([webpath == ignore for ignore in ['/favicon.ico', '/apple-touch-icon-precomposed.png', '/apple-touch-icon.png', '/apple-touch-icon-120x120-precomposed.png', '/apple-touch-icon-120x120.png']]) and
            not any([webpath.startswith(prefix) for prefix in webpath_allowlist])):
            self.send_error(404, 'Illegal webpath (' + this_line_number() + '): ' + webpath)
            return

        filepath = remove_leading_slash(webpath)
        self.path = filepath

        if webpath in [config_query_webpath, asset_query_webpath, game_query_webpath, doc_query_webpath, script_query_webpath, update_progress_query_webpath]:
            response_obj = {}
            if webpath == update_progress_query_webpath:
                if update_thread:
                    status = update_thread.get_status()
                    response_obj['status'] = status
                    response_obj['done'] = (status != 'Running')
                    response_obj['restartServer'] = (status == 'Done. Restart server.')
                else:
                    response_obj['status'] = 'No update in progress.'
                    response_obj['restartServer'] = False
                    response_obj['done'] = True

            elif webpath == config_query_webpath:
                if not args.kiosk: response_obj['IDE_USER'] = os.environ['USERNAME' if isWindows else 'USER']
                response_obj['rootPath'] = server_root_filepath
                response_obj['hasFinder'] = isWindows or isMacOS

                if isWindows or isMacOS:
                    applications = []
                    response_obj['applications'] = applications
                    potential_applications = workjson.load(os.path.join(quad_filepath, 'console/external-applications.json'))['Windows' if isWindows else 'macOS']
                    for app in potential_applications:
                        for path in app['paths']:
                            path = os.path.expandvars(os.path.expanduser(path))
                            if os.path.exists(path):
                                # Found this app
                                applications.append({'name': app['name'], 'path': path, 'types': app['types']})
                                #print('FOUND', path)
                                break
                            #else:
                            #    print('Did not find', path)

            elif webpath == script_query_webpath:
                aux_webpath = query[query.index('=') + 1:]
                # Security: check if aux_webpath has a prefix in webpath_allowlist
                if not any([aux_webpath.startswith(prefix) for prefix in webpath_allowlist]):
                    self.send_error(404, 'Illegal webpath (' + this_line_number() + '): ' + aux_webpath)
                    return

                # List the pyxl files
                aux_filepath = remove_leading_slash(aux_webpath)
                response_obj = glob.glob(aux_filepath + '**/*.pyxl', recursive=True)

                # Remove anything in a magic directory
                response_obj = [f for f in response_obj if not self.special_dir_regex.search(f)]

                # Sort all files
                response_obj.sort()

                # List additional files from the built-in scripts folder
                response_obj += ['quad://' + f[len(quad_filepath):] for f in sorted(glob.glob(quad_filepath + 'scripts/*.pyxl'))]

                # Fix slashes on windows
                if isWindows: response_obj = [f.replace('\\', '/') for f in response_obj]

            elif webpath == doc_query_webpath:
                aux_webpath = query[query.index('=') + 1:]

                # Security: check if aux_webpath has a prefix in webpath_allowlist
                if not any([aux_webpath.startswith(prefix) for prefix in webpath_allowlist]):
                    self.send_error(404, 'Illegal webpath: (' + this_line_number() + ')' + aux_webpath)
                    return

                aux_filepath = remove_leading_slash(aux_webpath)
                # List the files
                response_obj =  glob.glob(aux_filepath + '**/*.html', recursive=True)
                response_obj += glob.glob(aux_filepath + '**/*.md',   recursive=True)
                response_obj += glob.glob(aux_filepath + '**/*.pdf',  recursive=True)
                response_obj += glob.glob(aux_filepath + '**/*.txt',  recursive=True)

                # Do not remove any magic directory files; we want journals and such in this list

                # Sort all files
                response_obj.sort()

                # Fix slashes on windows
                if isWindows: response_obj = [f.replace('\\', '/') for f in response_obj]

            elif webpath == asset_query_webpath:
                aux_webpath = query[query.index('=') + 1:]

                # Security: check if aux_webpath has a prefix in webpath_allowlist
                if not any([aux_webpath.startswith(prefix) for prefix in webpath_allowlist]):
                    self.send_error(404, 'Illegal webpath: (' + this_line_number() + ')' + aux_webpath)
                    return

                # Raw files to exclude because they are already in use. Exclude
                # the files from quadplay's own metadata automatically.
                exclude_table = {}
                for f in ['label64.png', 'label128.png', 'preview.png']:
                    exclude_table[remove_leading_slash(aux_webpath) + f] = True

                # Search (quad_filepath | aux_webpath) + type + 's/*.' + type + '.json'
                asset_type_array = ['font', 'sprite', 'sound', 'map', 'data']
                for t in asset_type_array:
                    # Get the json files from the game directory
                    response_obj[t] = glob.glob(remove_leading_slash(aux_webpath + '**/*.' + t + '.json'), recursive=True)

                    # Look in each of the files for the URLs of raw files to exclude
                    for file in response_obj[t]:
                        try:
                            tmp = workjson.load(file)
                            if 'url' in tmp:
                                exclude_table[remove_leading_slash(aux_webpath) + tmp['url']] = True
                        except:
                            maybe_print('Warning: Could not parse ' + file)

                raw_extension = {'sprite': 'png', 'font': 'png', 'sound': 'mp3', 'map': 'tmx'}
                for t in asset_type_array:
                    # Get raw files
                    if t != 'data':
                        raw_files = glob.glob(remove_leading_slash(aux_webpath + '**/*.' + raw_extension[t]), recursive=True)
                    else:
                        raw_files = []

                    # Remove from raw_files anything that is in exclude_table
                    raw_files = [f for f in raw_files if not f in exclude_table]

                    # Sort all files from the game directory
                    response_obj[t] = sorted(response_obj[t] + raw_files)

                    # Remove anything in a magic directory
                    response_obj[t] = [f for f in response_obj[t] if not self.special_dir_regex.search(f)]

                    # Get the json files from the quadplay directory,
                    # sort, and append them to the list
                    response_obj[t] += ['quad://' + f[len(quad_filepath):] for f in sorted(glob.glob(quad_filepath + t + 's/*.' + t + '.json'))]

                    # Fix slashes on windows
                    if isWindows: response_obj[t] = [f.replace('\\', '/') for f in response_obj[t]]

            elif webpath == game_query_webpath:

                # Detect games with the same name as the folder
                redundant_pattern = re.compile(r'(?P<name>/[^/]+)(?P=name)\.game\.json$')
                set = {'examples' : quad_filepath + 'examples',
                       'builtins' : quad_filepath + 'games',
                       'mine'     : my_games_filepath}

                # For alpha testers and internal development
                if os.path.exists(quad_filepath + '_alpha'):
                    set['alpha'] = quad_filepath + '_alpha'
                    set['tests'] = quad_filepath + '_tests'

                for key, path in set.items():
                    # Do not change case on Windows (because this will be part of the json file that is read
                    # on multiple platforms) but *do* fix slashes
                    list = [{'url': path.replace('\\', '/'), 'title': 'TBD'} for path in sorted(glob.glob(path + '/**/*.game.json', recursive=True))]
                    for entry in list:
                        path = entry['url']

                        if key == 'mine':
                            # Make relative to the web root
                            entry['url'] = '/' + entry['url']
                        else:
                            # Shorten built-ins to quad://
                            entry['url'] = 'quad://' + path[len(quad_filepath):]

                        # Shorten games with redundant names to just the directory
                        entry['url'] = redundant_pattern.sub(r'\1/', entry['url'])
                        # Fetch the titles
                        with open(path, 'rt', encoding='utf8') as f:
                            game = workjson.loads(f.read())
                            entry['description'] = game['description'] if 'description' in game else ''
                            entry['title'] = game['title']

                    response_obj[key] = list

            response = json.dumps(response_obj, separators = (',', ':'));
            self.send_response(200)
            self.send_header('Content-type', 'text/json')
            self.send_header('Content-length', len(response))
            self.end_headers()
            self.wfile.write(response.encode('utf8'))

        elif re.search(self.silent_404_file_regex, webpath) and not os.path.exists(filepath):
            # Check to see if this file does not exist. If it does
            # not, return the 404 error but do not print to the terminal
            self.send_response(404)
            self.send_header('Content-type', 'text/json')
            self.send_header('Content-length', 0)
            self.end_headers()
        else:
            # Serve the file
            f = self.send_head()
            if f:
               try:
                   self.copyfile(f, self.wfile)
               except BrokenPipeError:
                   maybe_print('Broken pipe while loading', filepath)
               finally:
                   f.close()

    def log_request(self, code = '-', size = '-'):
        # Overridden to not show all requests
        #self.log_message('"%s" %s %s', self.requestline, str(code), str(size))
        pass

    def log_error(self, format, *args):
        # Overridden to allow a debugging point
        self.log_message(format, *args)

    def translate_path(self, path):
        if path == '/favicon.ico':
            # Browsers sometimes ask for the icon from this
            # location. Send them to the quadplay directory instead
            return quad_filepath + '/console/icons/favicon.ico'

        elif path == '/apple-touch-icon-precomposed.png' or path == '/apple-touch-icon.png':
            # Safari asks for the icon via this path
            return quad_filepath + '/console/icons/favicon-64x64.png'

        elif path == '/apple-touch-icon-120x120-precomposed.png' or path == '/apple-touch-icon-120x120.png':
            # Safari asks for the icon via this path
            return quad_filepath + '/console/icons/favicon-120x120.png'

        elif isWindows and server_root_filepath == '':
            # Need the whole path, since the "cwd" is wrong
            return self.path.split('?')[0]
        else:
            # Intentionally use Python 2 super syntax for potential
            # backward compatibility
            result = SimpleHTTPRequestHandler.translate_path(self, path)
            return result

if isWindows and args.gamepath:
    # Make drive letters canonical
    if args.gamepath[1] == ':':
        args.gamepath = args.gamepath[0].upper() + args.gamepath[1:]

# Makes a path relative to the server_root_filepath, and thus "absolute"
# for the web server root (not for the filesystem)
def platform_www_abspath(p):
    if isWindows:
        p = p[len(server_root_filepath):]
        if not p.endswith('.game.json'): p = os.path.join(p, '')
        t = os.path.normcase(p).replace('\\', '/')
        if t[0] != '/': t = '/' + t
        return t
    else:
        if server_root_filepath != '/':
            p = p[len(server_root_filepath)+1:]

        if p == '':
            return ''
        elif p.endswith('.game.json'):
            return os.path.join('/', p)
        else:
            return os.path.join('/', p, '')

# Process paths at top level so that they can be inherited by ThreadingHTTPServer
quad_filepath = canonicalize_filepath(os.path.join(os.path.dirname(__file__), '..'))
token_absfilepath = os.path.join(quad_filepath, 'tools/token.txt')

# Compute the serving paths
game_filepath = canonicalize_filepath(args.gamepath)
if game_filepath.endswith('.game.json'): game_filepath = os.path.dirname(game_filepath)
webpath_allowlist = [quad_filepath, game_filepath, my_games_filepath]

# Common ancestor
server_root_filepath = os.path.dirname(os.path.commonprefix([os.path.join(p, '') for p in webpath_allowlist]))

# Strip the common prefix
webpath_allowlist = [platform_www_abspath(p) for p in webpath_allowlist]
my_games_filepath = remove_leading_slash(my_games_filepath[len(server_root_filepath):])

# Remove trailing slash
asset_query_webpath = webpath_allowlist[0]
if len(asset_query_webpath) > 0 and asset_query_webpath[-1] == '/': asset_query_webpath = asset_query_webpath[:-1]
game_query_webpath = asset_query_webpath + '/console/games.json'
config_query_webpath = asset_query_webpath + '/console/_config.json'
script_query_webpath = asset_query_webpath + '/console/_scripts.json'
update_progress_query_webpath = asset_query_webpath + '/console/_update_progress.json'
doc_query_webpath = asset_query_webpath + '/console/_docs.json'
quad_filepath = remove_leading_slash(webpath_allowlist[0])
asset_query_webpath += '/console/_assets.json'

maybe_print = ignore if args.quiet else print

if False:
    # Debug paths
    maybe_print("\nserver_root_filepath   = '" + server_root_filepath + "'")
    maybe_print("quad_filepath          = '" + quad_filepath + "'")
    maybe_print("my_games_filepath      = '" + my_games_filepath + "'")
    maybe_print("asset_query_webpath    = '" + asset_query_webpath + "'")
    maybe_print("script_query_webpath   = '" + script_query_webpath + "'")
    maybe_print("doc_query_webpath      = '" + doc_query_webpath + "'")
    maybe_print("game_query_webpath     = '" + game_query_webpath + "'")
    maybe_print("config_query_webpath   = '" + config_query_webpath + "'")
    maybe_print('webpath_allowlist      = ', webpath_allowlist, '\n')
    sys.exit()

# SSL support doesn't work yet due to self-signed certificate problems
useSSL = False

# Runs in a different process
def launchServer(post_token):
    global httpd, args, server_root_filepath, token

    # Assign back to the global token. We have to do this because
    # Python reinitializes all globals
    token = post_token

    # Serve from the common directory, unless the common directory
    # is the filesystem root and we're on windows where that cannot
    # be the cwd
    old_path = os.getcwd()
    if not isWindows or server_root_filepath != '':
        os.chdir(server_root_filepath)

    attempts = 0
    while attempts < 2:
        try:
            attempts += 1
            # '' = 0.0.0.0 = all local IP addresses, needed for
            # supporting devices other than just localhost
            server_address = ('' if args.serve else 'localhost', args.port)
            httpd = QuadplayHTTPServer(server_address, QuadplayHTTPRequestHandler)

            if useSSL:
                # This path is never used in the current implementation
                httpd.socket = ssl.wrap_socket(httpd.socket,
                                               keyfile='console/ssl/local-key.pem',
                                               certfile='console/ssl/local-cert.pem',
                                               do_handshake_on_connect=False,
                                               server_side=True)
            httpd.serve_forever()
            os.chdir(old_path)
            return

        except OSError as e:
            if attempts < 2 and e.errno == 48:
                print('Cleaning up previous quadplay server instance...')

                # Read the old token
                with open(token_absfilepath, 'r') as file: old_token = file.readline()

                # Send a QUIT command via POST
                try:
                    urllib.request.urlopen(urllib.request.Request(
                        'http://localhost:' + str(server_address[1]) + '/',
                        bytes('{"command":"quit","token":"' + old_token + '"}', 'utf-8'),
                        {'Content-Type': 'application/json;charset=UTF-8'}))
                except:
                    # The server on the other side will terminate immediately
                    # (or, it might be unresponsive!)
                    pass

                # Wait for the OS to clean up the listener socket
                time.sleep(0.1)

                # The loop will now make a second attempt
            else:
                maybe_print(e)
                maybe_print('Not starting a local server, since one is already running.');
                os.chdir(old_path)

#######################################################################

def main():
    global webpath_allowlist, server_root_filepath, token

    token = "%0.7X" % random.randrange(0, 0x10000000)

    if not args.quiet:
        if isWindows: os.system('color')
        banner = ['',
            '\033[91m                      ╷       ╷                    ',
            '\033[95m╭───╮ ╷   ╷  ───╮ ╭── │ ╭───╮ │   ───╮ ╷   ╷   ▒▒  ',
            '\033[94m│   │ │   │ ╭── │ │   │ │   │ │  ╭── │ │   │ ▒▒  ▒▒',
            '\033[96m╰── │ ╰───┘ ╰───╯ ╰───╯ │ ──╯ ╰─ ╰───╯ ╰── │   ▒▒  ',
            '\033[37m    ╵                   ╵                ──╯       ','\033[0m']

        for line in banner:
            maybe_print('         ' + line)

    maybe_print('_________________________________________________________________________\n')
    maybe_print('quadplay version ' + installed_quadplay_version['text'] + ' from ' + quadplay_origin + '\n')

    myip = '127.0.0.1'
    if args.serve:
        try:
            maybe_print('Getting IP address...')
            myip = ''

            # just try this first on mac because the other approaches fail so frequently
            if isMacOS:
               myip = (os.popen('ipconfig getifaddr en0').read().strip() or
                       os.popen('ipconfig getifaddr en1').read().strip())
            if not myip: myip = socket.gethostbyname(socket.gethostname())
        except:
            if isMacOS:
                # The above can fail apparently randomly on MacOS (https://bugs.python.org/issue35164)
                # and that has happened to us. This is a workaround:
                maybe_print('gethostbyname failed due to a known MacOS internal error. Falling back to\nipconfig...')

                myip = os.popen('ipconfig getifaddr en0').read().strip()
                if not myip:
                   maybe_print('Could not find en0. Looking for en1...')
                   myip = os.popen('ipconfig getifaddr en1').read().strip()

                if not myip:
                    maybe_print('WARNING: ipconfig could not find a valid en0 or en1 adapter. Server IP address is unknown.')
                    myip = '127.0.0.1'
                else:
                    maybe_print('WARNING: gethostbyname unexpectedly failed. Server IP address is unknown.')
                    myip = '127.0.0.1'

    url = 'http' + ('s' if useSSL else '')+ '://' + myip + ':' + str(args.port)
    if not quad_filepath or quad_filepath[0] != '/': url += '/'
    url += os.path.join(quad_filepath, 'console/app.html?fastReload=1&token=' + token)

    if args.nativeapp: url += '&nativeapp=1'
    if not args.noupdatecheck and not args.kiosk:
        if quadplay_origin == 'development git clone':
            url += '&update=dev'
        elif quadplay_origin == 'git clone':
            url += '&update=git'
        else:
            url += '&update=1'

    # Sanitized username
    url += '&name=' + re.compile('[^A-Za-z0-9_]').sub('', getpass.getuser())

    if args.kiosk:
        url += '&mode=Maximal&kiosk=1'
    elif args.play:
        url += '&mode=Windowed'
    else:
        url += '&IDE=1'

    if args.offline:
        url += '&offline=1'

    if args.gamepath != '':
        t = args.gamepath
        if not t.startswith('http://') and not t.startswith('quad://'):
            t = platform_www_abspath(os.path.abspath(t))
            if t and not isWindows and t[0] != '/': t = '/' + t
        url += '&game=' + t

    if args.serve:
        # Do not support POST in host mode
        maybe_print('Your firewall may need to be configured to load on other devices.\n')
    else:
        url += '&quadserver=1'

    maybe_print('\nServing from:\n\n   ' + url + '\n')

    # Run from the quadplay path
    maybe_print('   quad://  = ' + os.path.join(server_root_filepath, quad_filepath))
    maybe_print('   My games = ' + os.path.join(server_root_filepath, my_games_filepath) + '/')
    maybe_print('   cwd      = ' + remove_trailing_slash(server_root_filepath))
    maybe_print('   args     = ' + str(args), '\n')

    if not args.nativeapp:
        try:
            # For kbhit()
            import msvcrt
        except ImportError:
            import termios, atexit, select
            stdinFd = sys.stdin.fileno()
            new_term = termios.tcgetattr(stdinFd)
            old_term = termios.tcgetattr(stdinFd)
            def set_normal_term():
                termios.tcsetattr(stdinFd, termios.TCSAFLUSH, old_term)

            # New terminal setting unbuffered
            new_term[3] = (new_term[3] & ~termios.ICANON & ~termios.ECHO)
            termios.tcsetattr(stdinFd, termios.TCSAFLUSH, new_term)

            # Support normal-terminal reset at exit
            atexit.register(set_normal_term)

        # Platform independent keyboard key press detection
        def kbhit():
            if isWindows:
                return msvcrt.kbhit()
            else:
                dr, dw, de = select.select([sys.stdin], [], [], 0)
                return dr != []

    httpd = 0

    serverThread = multiprocessing.Process(target=launchServer, args=[token])
    maybe_print('Starting local server thread...')
    serverThread.start()
    time.sleep(1.5)

    if not serverThread.is_alive():
        print('Could not start quadplay server.')
        sys.exit(2)
    else:
        # Store the token so that other servers can force-quit this one
        with open(token_absfilepath, 'wt') as file: file.write(token)

    browser_filepath = None

    # Try to find Edge, then Chromium if they exist because those
    # are the fastest quadplay browsers and support extra kiosk
    # features.
    if isWindows:
        import winreg
        for p in ['MSEdgeHTM', 'ChromeHTML', 'BraveHTML']:
            try:
                command = winreg.QueryValueEx(winreg.OpenKey(winreg.HKEY_CLASSES_ROOT, p + "\\shell\\open\\command", 0, winreg.KEY_READ), "")[0]
                browser_filepath = re.search("\"(.*?)\"", command).group(1)
                if os.path.exists(browser_filepath):
                    break
                else:
                    browser_filepath = None
            except:
                pass
    else:
        if isMacOS:
            browser_list = ['/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge', '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser']
        else: # Linux
            browser_list = ['/usr/bin/chromium-browser']

        for p in browser_list:
            if os.path.exists(p):
                browser_filepath = p
                break

    # Require quotes around URLs so that the shell does not interpret
    # "&" in a URL as a shell option to run in another thread
    #
    # The --start-fullscreen argument works for Edge and Chrome on
    # Windows and macOS. There is no direct command line way to launch
    # Safari or Firefox in fullscreen mode, or to launch in fullscreen
    # using xdg-open on Linux.

    browser_command = ''

    # Monitor the browser and close the server when it exits
    auto_close_server = False

    # We must force a fake user data directory to prevent
    # the chromium script from sharing an existing session, which would cause the
    # call() below to not block: https://askubuntu.com/questions/35392/how-to-launch-a-new-instance-of-google-chrome-from-the-command-line
    # This is also intended to keep processes clear of each other when running multiple instances of quadplay.
    chromium_nativeapp_args = ' --user-data-dir="' + os.path.join(os.path.expanduser('~'), 'quadplay_session') + '" --no-user-gesture-required '
    quadplay_session = os.path.join(os.path.expanduser('~'), 'quadplay_session')
    if not os.path.exists(quadplay_session): os.mkdir(quadplay_session)

    if args.gamepath == '':
        maybe_print('\nLoading default game. You can supply the URL or local relative path to your game on\nthe command line, for example "tools/quadplay-server foo/mygame", to load it directly.\n')

    if isWindows:
        if browser_filepath and args.nativeapp:
            # Use Chrome/Edge if available, in order to present more
            # as a native app.  On recent Windows 10, we can count on
            # Edge being present.
            #
            # Because of the way that windows CMD processes quotes in
            # the command line, it is necessary to put double quotes
            # around the ENTIRE line as well as around the individual
            # arguments when using os.system(), but not for
            # subprocess.call():
            #
            # https://superuser.com/questions/238810/problem-with-quotes-around-file-names-in-windows-command-shell
            browser_command = '"' + browser_filepath + '" -app="' + url + '" ' + chromium_nativeapp_args + (' --start-fullscreen' if args.kiosk else '')
            auto_close_server = True
        else:
            browser_command = 'start "quadplay" "' + url + '"' + (' --start-fullscreen' if args.kiosk else '')
    elif isMacOS:
        if browser_filepath and args.nativeapp:
            auto_close_server = True
            browser_command = '"' + browser_filepath + '" ' + chromium_nativeapp_args + ' -app="' + url + '" ' + (' --start-fullscreen' if args.kiosk else '')
        else:
            browser_command = 'open "' + url + '" ' + (' --args --start-fullscreen' if args.kiosk else '')
    else: # Linux
        if browser_filepath and args.nativeapp:
            browser_command = browser_filepath + ' -app="' + url + '" ' + chromium_nativeapp_args + (' --start-fullscreen' if args.kiosk else '')
            auto_close_server = True
        else:
            browser_command = 'xdg-open "' + url + '"'

    if auto_close_server:
        # Block until the client terminates, but do so in a way that
        # we can interrupt if the server self-terminates first on
        # macOS.
        maybe_print('Launching IDE via ' + browser_command)
        client = subprocess.Popen(shlex.split(browser_command), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL if args.quiet else None)

        # If another browser process is already running (on Edge Chromium, at least), then
        # the client process may terminate instantly because it is a zygote. Require an explicit
        # close message in this case, which quadplay will send to the server.
        auto_close_with_client = False
        while (not auto_close_with_client or client.poll() == None) and serverThread.is_alive():
            time.sleep(0.2)

        if serverThread.is_alive():
            maybe_print('Client IDE process auto terminated')
        else:
            maybe_print('Server thread auto terminated')

        # Kill the client if it didn't already terminate
        client.terminate()
    else:
        os.system(browser_command)
        # Wait for key stroke
        maybe_print('\n**Press any key to terminate the server when done**\n')
        while not kbhit() and serverThread.is_alive(): time.sleep(0.25)

    maybe_print('\nShutting down...')

    # Attempt to gracefully shut down
    if httpd: threading.Thread(target=lambda : httpd.shutdown())
    time.sleep(1)
    serverThread.terminate()

    return 0

if __name__ == "__main__":
    sys.exit(main())


import os
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
from transformers import RobertaTokenizer, RobertaForSequenceClassification
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from transformers import GPT2Tokenizer, GPT2LMHeadModel
from transformers import AutoTokenizer, AutoModel
from transformers import AutoTokenizer, AutoModelForMaskedLM
from transformers import BartForConditionalGeneration, BartTokenizer
import click
from pathlib import Path
from nlgeval.metrics.sources.rouge_we_utils import load_embeddings_from_web
from nlgeval.metrics.sources.s3_utils.s3_utils import load_model

@click.command()
@click.option('--model_name', default=None)
@click.option('--scorer_name', default=None)
@click.option('--dataset_name', default=None)
@click.option('--metric_name', default=None)
@click.option('--datset_options', default=None)
@click.option('--save_dir', default="pretrained")
@click.option('--save_name', default="saved")
def run(dataset_name, scorer_name, metric_name, datset_options, model_name, save_dir, save_name):
    if dataset_name is not None:
        dataset = load_dataset(dataset_name, datset_options)
        dataset.save_to_disk("./" + save_dir + "/" + save_name)
    elif model_name is not None:
        tokenizer = AutoTokenizer.from_pretrained(model_name)
        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
        model.save_pretrained("./" + save_dir + "/" + save_name + "-model")
        tokenizer.save_pretrained("./" + save_dir + "/" + save_name + "-tokenizer")
    elif scorer_name is not None:
        if scorer_name == "grammar":
            tokenizer = RobertaTokenizer.from_pretrained("textattack/roberta-base-CoLA")
            model = RobertaForSequenceClassification.from_pretrained("textattack/roberta-base-CoLA", num_labels=2)
            model.save_pretrained("./" + save_dir + "/grammar-scorer-model")
            tokenizer.save_pretrained("./" + save_dir + "/grammar-scorer-tokenizer")
        elif scorer_name == "relevance":
            tokenizer = AutoTokenizer.from_pretrained("ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli")
            model = AutoModelForSequenceClassification.from_pretrained("ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli")
            model.save_pretrained("./" + save_dir + "/relevance-scorer-model")
            tokenizer.save_pretrained("./" + save_dir + "/relevance-scorer-tokenizer")
        elif scorer_name == "coherence":
            tokenizer = GPT2Tokenizer.from_pretrained("gpt2-medium")
            model = GPT2LMHeadModel.from_pretrained("gpt2-medium", return_dict=True)
            model.save_pretrained("./" + save_dir + "/coherence-scorer-model")
            tokenizer.save_pretrained("./" + save_dir + "/coherence-scorer-tokenizer")
        else:
            raise ValueError("Wrong scorer name")
    elif metric_name is not None:
        if metric_name == "mover_score":
            tokenizer = AutoTokenizer.from_pretrained("textattack/bert-base-uncased-MNLI", do_lower_case=True)
            model = AutoModel.from_pretrained("textattack/bert-base-uncased-MNLI", output_hidden_states=True, output_attentions=True)
            model.save_pretrained("./" + save_dir + "/mover_score")
            tokenizer.save_pretrained("./" + save_dir + "/mover_score")
        elif metric_name == "blanc":
            tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
            model = AutoModelForMaskedLM.from_pretrained("bert-base-uncased")
            tokenizer.save_pretrained("./" + save_dir + "/blanc")
            model.save_pretrained("./" + save_dir + "/blanc")
        elif metric_name == "bart_score":
            tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn")
            model = BartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn")
            tokenizer.save_pretrained("./" + save_dir + "/bart_score")
            model.save_pretrained("./" + save_dir + "/bart_score")
        elif metric_name == "rouge_we":
            load_embeddings_from_web(save_path="./" + save_dir + "/rouge_we")
        elif metric_name == "s3":
            load_model("./" + save_dir + "/s3_pyr", "https://drive.google.com/uc?id=19sdnH0e5YOtZBYi3kNQ-n0J8FRwuQhB2")
            load_model("./" + save_dir + "/s3_resp", "https://drive.google.com/uc?id=1qD-XfIiSocUi9QUR0ne4sn-vgmQSm3dV")
    else:
        raise ValueError("Nothing to save")

if __name__ == '__main__':
    run()


from typing import List

from searchapp.models.search_result.user_searchable_fields import UserSearchableFields
from searchapp.models.search_result.ticket_searchable_fields import TicketSearchableFields
from searchapp.models.search_result.user_result import UserResult
from searchapp.models.search_result.ticket_result import TicketResult
from searchapp.repository.user_repo import UserRepo
from searchapp.repository.ticket_repo import TicketRepo
class SearchService:

    def __init__(self, user_repo: UserRepo, ticket_repo: TicketRepo) -> None:
        self.user_repo = user_repo
        self.ticket_repo = ticket_repo

    def get_user_search_terms(self) -> UserSearchableFields:
        return UserSearchableFields(terms=["_id", *self.user_repo.indexes])

    def search_user(self, term: str, value: str) -> List[UserResult]:
        users = self.user_repo.search_by_term(term, value)
        return [
            UserResult(
                user,
                assigned_tickets=self.ticket_repo.search_by_term('assignee_id', str(user._id))
            ) for user in users
        ]

    def get_ticket_search_terms(self) -> TicketSearchableFields:
        return TicketSearchableFields(terms=["_id", *self.ticket_repo.indexes])

    def search_tickets(self, term: str, value: str) -> List[TicketResult]:
        tickets = self.ticket_repo.search_by_term(term, value)
        result: List[TicketResult] = []
        for ticket in tickets:
            users = self.user_repo.search_by_term('_id', str(ticket.assignee_id))
            result = [
                *result,
                TicketResult(
                    ticket,
                    assignee=users[0] if users else None
                )
            ]
        return result


# Cut EnMAP L2 tiles into patches and split into train and test set

import os
import glob
import pyproj
import rasterio
from tqdm import tqdm
from rasterio.warp import Resampling

from shapely.ops import transform

TILE_SIZE = 64
UPSCALE_FACTOR_ENMAP = 1  # 3 for 10m resolution with bilinear

ENMAP_PATH = "/ds2/remote_sensing/enmap/"
OUTPUT_DIR = "/ds2/remote_sensing/enmap_worldcover_dataset_v3/train/"
TESTFILES = "/ds2/remote_sensing/enmap_worldcover_dataset/testfiles.txt"

if __name__ == "__main__":
    wgs84 = pyproj.CRS("EPSG:4326")

    l2_product_dirs = [
        x
        for x in glob.glob(os.path.join(ENMAP_PATH, "*", "*", "*", "*L2A-DT*"))
        if os.path.isdir(x)
    ]
    l2_spectral_products = [
        glob.glob(os.path.join(d, "*SPECTRAL_IMAGE.TIF"))[0] for d in l2_product_dirs
    ]
    l2_metadata = [
        glob.glob(os.path.join(d, "*METADATA.XML"))[0] for d in l2_product_dirs
    ]
    print(f"Found {len(l2_spectral_products)} EnMAP products.")

    # make sure that there are no duplicate enmap files
    filenames = [x.split("/")[-1] for x in l2_spectral_products]
    assert len(filenames) == len(set(filenames))

    with open(TESTFILES) as f:
        testfiles = [x.strip() for x in f.readlines()]

    for enmap_product in l2_spectral_products:
        filename = enmap_product.split("/")[-1].split(".TIF")[0]
        outdir = os.path.join(OUTPUT_DIR, filename)

        if filename in testfiles:
            outdir = outdir.replace("train", "test")

        if os.path.exists(outdir):
            print("Directory already exists/EnMAP file already processed")
            save = False
            continue
        else:
            os.mkdir(outdir)
            save = True

        with rasterio.open(enmap_product) as dataset:
            # resample data to target shape
            enmap_meta = dataset.meta.copy()
            enmap_meta["bounds"] = dataset.bounds
            enmap = dataset.read(
                out_shape=(
                    dataset.count,
                    int(dataset.height * UPSCALE_FACTOR_ENMAP),
                    int(dataset.width * UPSCALE_FACTOR_ENMAP),
                ),
                resampling=Resampling.bilinear,
            )

            # scale image transform
            transform = dataset.transform * dataset.transform.scale(
                (dataset.width / enmap.shape[-1]), (dataset.height / enmap.shape[-2])
            )
            enmap_meta["transform"] = transform
            enmap_meta["width"] = enmap.shape[-1]
            enmap_meta["height"] = enmap.shape[-2]

        # create tiles from uncorrupted pixels
        tiles = []
        for i in range(0, enmap.shape[1], TILE_SIZE):
            for j in range(0, enmap.shape[2], TILE_SIZE):
                if i + TILE_SIZE > enmap.shape[1] or j + TILE_SIZE > enmap.shape[2]:
                    continue

                enmap_tile = enmap[:, i : i + TILE_SIZE, j : j + TILE_SIZE]

                if (enmap_tile == enmap_meta["nodata"]).mean(axis=(1, 2)).all():
                    # all bands are nodata for every pixel
                    continue

                tiles.append(enmap_tile)

        print(f"Number of valid tiles for {enmap_product.split('/')[-1]}: {len(tiles)}")

        if save:
            for idx, tile in tqdm(enumerate(tiles), total=len(tiles)):
                with rasterio.open(
                    os.path.join(outdir, f"tile{idx}_enmap.tif"),
                    "w",
                    driver="GTiff",
                    nodata=-32768.0,
                    dtype=tile.dtype,
                    count=tile.shape[0],
                    width=tile.shape[2],
                    height=tile.shape[1],
                ) as f:
                    f.write(tile)
        else:
            print("Not saved, see above")


from typing import Tuple

import numpy as np
from scipy.spatial import ConvexHull

from .Base import Base_Space, Base_Solution, Base_Optimiser

class Base_Scout(object):
    __opt: Base_Optimiser
    __space: Base_Space
    __uniqueness: int   # set to zero to disable testing for uniqueness; otherwise, number of attempts
    __sigma: np.double

    def __init__(self) -> None:
        self.__opt = None
        self.__space = None
        self.__uniqueness: int = 100
        self.__sigma = 1.0

    @property
    def scouts(self) -> int:
        return 1

    @property
    def optimiser(self) -> Base_Optimiser:
        return self.__opt

    @optimiser.setter
    def optimiser(self, opt: Base_Optimiser) -> None:
        self.__opt = opt
        self.__space = opt.space

    @property
    def space(self) -> Base_Space:
        return self.__space

    @property
    def uniqueness(self) -> int:
        return self.__uniqueness

    @uniqueness.setter
    def uniqueness(self, timeout: int) -> None:
        if timeout <= 0:
            self.__uniqueness = 0
        else:
            self.__uniqueness = timeout

    @property
    def sigma(self) -> np.double:
        return self.__sigma

    @sigma.setter
    def sigma(self, value: np.double) -> None:
        self.__sigma = value

    def __new_unique_scout(self, near: Base_Solution = None, sigma: np.double = 1) -> Base_Solution:
        count = 0
        while True:
            S = Base_Solution(self.__space)
            if self.__uniqueness == 0:
                break
            if self.__opt.lookup(S) is None:
                break
            count = count + 1
            if count == self.__uniqueness:
                S = None
                break
        return S

    def scout(self) -> Tuple[Base_Solution, np.uint32]:
        S = self.__new_unique_scout()
        if S is None:
            print(" - Base_Scout::scout] Timeout searching for scout location (bailing).")
        if S is None:
            return None, 0xFFFFFFFF

        rank = self.__opt.evaluate_and_record(S)
        if self.__opt.noisy and rank == 0:
            print("B", end="") # print("    Base_Scout: {cst}".format(cst=S.cost))
        return S, rank

class FrontierScout(Base_Scout):
    def __init__(self) -> None:
        Base_Scout.__init__(self)

    def __new_unique_scout(self, near: Base_Solution) -> Base_Solution:
        count = 0
        while True:
            S = Base_Solution(self.space, self.space.random_nearby_coordinate(near.coordinate, self.sigma))

            if self.uniqueness == 0:
                break
            if self.optimiser.lookup(S) is None:
                break
            count = count + 1
            if count == self.uniqueness:
                S = None
                break
        return S

    def scout(self) -> Tuple[Base_Solution, np.uint32]:
        S = None

        near = self.optimiser.select_solution(10)
        if near is None:
            print(" - FrontierScout::scout] Timeout getting nearby scout location.")
        else:
            S = self.__new_unique_scout(near)
            if S is None:
                print(" - FrontierScout::scout] Timeout searching for nearby scout location.")
        if S is None:
            return super().scout()

        rank = self.optimiser.evaluate_and_record(S)
        if self.optimiser.noisy and rank == 0:
            print("F", end="") # print("    FrontierScout: {cst}".format(cst=S.cost))
        return S, rank

class CascadeScout(FrontierScout):
    def __init__(self) -> None:
        Base_Scout.__init__(self)

    def __new_unique_scout(self) -> Base_Solution:
        cascade = self.optimiser.cascade
        if cascade is None:
            return None
        hull = cascade.hull
        if hull is None:
            return None

        origin_index = len(cascade.pts) - 1
        count = 0
        while True:
            if count == self.uniqueness:
                S = None
                break
            count = count + 1

            simplex = hull.simplices[self.space.rng.choice(len(hull.simplices))]
            sp1 = self.space.rng.choice(len(simplex))
            sp2 = sp1 + 1
            if sp2 == len(simplex):
                sp2 = 0
            iv1 = simplex[sp1]
            if iv1 == origin_index:
                continue
            iv2 = simplex[sp2]
            if iv2 == origin_index:
                continue

            S1 = cascade.sols[iv1]
            S2 = cascade.sols[iv2]

            delta = self.space.delta(S1.coordinate, S2.coordinate)
            S = Base_Solution(self.space, self.space.random_nearby_coordinate(S1.coordinate, self.sigma, delta))

            if self.uniqueness == 0:
                break
            if self.optimiser.lookup(S) is None:
                break
        return S

    def scout(self) -> Tuple[Base_Solution, np.uint32]:
        S = self.__new_unique_scout()
        if S is None:
            return super().scout()

        rank = self.optimiser.evaluate_and_record(S)
        if self.optimiser.noisy and rank == 0:
            print("C", end="") # print("    CascadeScout: {cst}".format(cst=S.cost))
        return S, rank

class BA_Patch(Base_Scout):
    __Nbee: int
    __level: int
    __l_max: int
    __scale: np.double
    __best: Base_Solution

    def __init__(self, number_of_bees: int, level_max: int) -> None:
        Base_Scout.__init__(self)
        self.__Nbee = number_of_bees
        self.__level = -1
        self.__l_max = level_max
        self.__scale = 1
        self.__best = None
        assert number_of_bees > 0, "Number of bees in patch must be a positive integer"
        assert level_max > 0, "Maximum level must be a positive integer"

    @property
    def scouts(self) -> int:
        return self.__Nbee

    def __new_unique_scout(self) -> Base_Solution:
        count = 0
        while True:
            S = Base_Solution(self.space, self.space.random_nearby_coordinate(self.__best.coordinate, self.sigma * self.__scale))

            if self.uniqueness == 0:
                break
            if self.optimiser.lookup(S) is None:
                break
            count = count + 1
            if count == self.uniqueness:
                S = None
                break
        return S

    def scout(self) -> Tuple[Base_Solution, np.uint32]:
        S = None

        Nfails = 0
        for b in range(0, self.__Nbee):
            if self.__level == -1:
                self.__best = self.optimiser.select_solution(10)
                if self.__best is None:
                    if self.optimiser.noisy:
                        print(" - BA_Patch::scout: No starting solution available?")
                else:
                    self.__scale = 1
                    self.__level = 0
                    Nfails = 0

            if self.__best is None:
                S, rank = super().scout()
                continue

            S = self.__new_unique_scout()
            if S is None:
                if self.optimiser.noisy:
                    print(" - BA_Patch::scout: No nearby solution - resetting")
                self.__level = -1
                continue
            rank = self.optimiser.evaluate_and_record(S)
            if self.optimiser.noisy and rank == 0:
                print("[BA({l})]".format(l=self.__level), end="")

            old_cost = self.__best.cost
            new_cost = S.cost
            if np.any(new_cost < old_cost):
                self.__best = S
                Nfails = 0
            else:
                Nfails = Nfails + 1

        if Nfails == self.__Nbee:
            self.__level = self.__level + 1
            if self.__level == self.__l_max:
                self.__level = -1
            else:
                self.__scale = self.__scale / 2

        return S, rank


import requests
from bs4 import BeautifulSoup
from requests import ReadTimeout
import csv
import pandas as pd
import heapq

csv_file = open('tags.csv', 'a+', newline='')
csv_writer = csv.writer(csv_file)

path = r"D:\个人\学习\大三下\数据可视化\实验\大作业\ml-25m\genome-scores.csv"
path_tag=r"D:\个人\学习\大三下\数据可视化\实验\大作业\ml-25m\genome-tags.csv"
df = pd.read_csv(path, encoding='gbk', chunksize=1128)
df_tag=pd.read_csv(path_tag,encoding='gbk')
tags=df_tag.values.tolist()
# print(tags)

for chunk in df:
    lines =chunk.values
    # print(lines)
    scores=lines[:,2].tolist()
    # print(scores)
    max_scores=heapq.nlargest(5,scores)
    # print(max_scores)
    index_scores=list(map(scores.index,heapq.nlargest(5,scores)))
    # print(list(index_scores))
    #在tag中根据index搜索对应的tag
    newtags=[int(lines[0,0])]
    print("现在读取id为{}的电影标签".format(newtags[0]))
    for i in range(5):
        tag=tags[index_scores[i]][1]
        newtags.append(tag)
    #写入文件
    csv_writer.writerow(newtags)

csv_file.close()  # 关闭csv文件


import numpy as np
from numba import njit
from .core import MarkovChain
from .._gridtools import cartesian_nearest_index, cartesian

def estimate_mc(X):
    r"""
    Estimate the Markov chain associated with a time series :math:`X =
    (X_0, \ldots, X_{T-1})` assuming that the state space is the finite
    set :math:`\{X_0, \ldots, X_{T-1}\}` (duplicates removed). The
    estimation is by maximum likelihood. The estimated transition
    probabilities are given by the matrix :math:`P` such that
    :math:`P[i, j] = N_{ij} / N_i`, where :math:`N_{ij} =
    \sum_{t=0}^{T-1} 1_{\{X_t=s_i, X_{t+1}=s_j\}}`, the number of
    transitions from state :math:`s_i` to state :math:`s_j`, while
    :math:`N_i` is the total number of visits to :math:`s_i`. The result
    is returned as a `MarkovChain` instance.

    Parameters
    ----------
    X : array_like
        A time series of state values, from which the transition matrix
        will be estimated, where `X[t]` contains the t-th observation.

    Returns
    -------
    mc : MarkovChain
        A MarkovChain instance where `mc.P` is a stochastic matrix
        estimated from the data `X` and `mc.state_values` is an array of
        values that appear in `X` (sorted in ascending order).

    """
    X = np.asarray(X)
    axis = 0 if X.ndim > 1 else None
    state_values, indices = np.unique(X, return_inverse=True, axis=axis)

    n = len(state_values)
    P = np.zeros((n, n))  # dtype=float to modify in place upon normalization
    P = _count_transition_frequencies(indices, P)
    P /= P.sum(1)[:, np.newaxis]

    mc = MarkovChain(P, state_values=state_values)
    return mc

@njit(cache=True)
def _count_transition_frequencies(index_series, trans_counter):
    T = len(index_series)
    i = index_series[0]
    for t in range(1, T):
        j = index_series[t]
        trans_counter[i, j] += 1
        i = j
    return trans_counter

def fit_discrete_mc(X, grids, order='C'):
    r"""
    Function that takes an arbitrary time series :math: `(X_t)_{t=0}^{T-1}` in
    :math: `\mathbb R^n` plus a set of grid points in each dimension and converts
    it to a MarkovChain by first applying discretization onto the grid
    and then estimation of the Markov chain.

    Parameters
    ----------

    X: array_like(ndim=2)
        Time-series such that the t-th row is :math:`x_t`.
        It should be of the shape T x n, where n is the number of dimensions.

    grids: array_like(array_like(ndim=1))
        Array of `n` sorted arrays. Set of grid points in each dimension

    Examples
    --------

    >>> grids = (np.arange(3), np.arange(2))
    >>> X = [(-0.1, 1.2), (2, 0), (0.6, 0.4), (1.0, 0.1)]
    >>> mc = fit_discrete_mc(X, grids)
    >>> mc.state_values
    array([[0, 1],
           [1, 0],
           [2, 0]])
    >>> mc.P
    array([[0., 0., 1.],
           [0., 1., 0.],
           [0., 1., 0.]])

    Returns
    -------

    mc: MarkovChain
        An instance of the MarkovChain class constructed after discretization
        onto the grid.
    """
    X_indices = cartesian_nearest_index(X, grids, order=order)
    mc = estimate_mc(X_indices)
    # Assign the visited states in the cartesian product as the state values
    prod = cartesian(grids, order=order)
    mc.state_values = prod[mc.state_values]
    return mc


import numpy as np
from numpy.random import seed as randseed
import os
from config import params
import scipy.io as sio
from math import pi

s = 20001 # random seed
snr = 100
params.fname = str(snr) + 'dB/' # redo fname for SNR
snap = 10
nsamples = len(params.theta)
Nfeat = params.Nsensor*(params.Nsensor+1)#2*params.Nsensor**2#
seed = np.arange((nsamples))
filename = '_5track_' + str(snap) + 'snaps'

# Define source angles
params.Nsources = 5
Amp = [1,1,1,1,1]
theta = np.zeros((params.Nsources, len(params.theta)))
theta[0,:] = params.theta[np.arange(0, len(params.theta))]
theta[1,:] = params.theta[np.arange(len(params.theta)-1,-1, -1)]
theta[2,:] = params.theta[22]*np.ones((len(params.theta),))
theta[3,:] = params.theta[60]*np.ones((len(params.theta),))
theta[4,:] = params.theta[170]*np.ones((len(params.theta),))

def generate_signal(theta, Amp, A, snr, snap, params):
    Xsource = np.zeros((snap,params.nclasses))
    Q = np.zeros((params.Nsources,), dtype=int)

    # Sparse indices for N sources
    for n in np.arange(params.Nsources):
        Q[n] = np.argmin(np.abs(params.theta - theta[n]))# index for source n
    Xsource[:,Q] = np.matlib.repmat(Amp, snap, 1)
    x = Xsource

    # random source phase
    if not params.Coh:
        phi = np.exp(-1j*2*pi*np.random.rand(snap, params.nclasses)-pi)
        Xsource = Xsource*phi/np.abs(phi)
    y = np.matmul(Xsource, A.T).T
    y = y/np.linalg.norm(y, axis=0)
    if not (snr==100):
        rnl = 10**(-snr/20)/np.sqrt(2)/np.sqrt(Nsensor) #note: sqrt(0.5) ensures the random variance is 1
        y = y + rnl*np.squeeze(np.random.randn(Nsensor, snap, 2).view(np.complex128))

    return y, x

def form_features(csdm):

    N = csdm.shape[1]
    iu1 = np.triu_indices(N)
    csdm_vec = csdm[iu1]
    feats = np.concatenate( (np.real(csdm_vec), np.imag(csdm_vec)),axis=-1 )

    return feats

## Make A matrix (plane wave weights)
Nsensor = len(params.xq)
u = np.sin(np.deg2rad(params.theta))
v = np.cos(np.deg2rad(params.theta))
A = np.exp(1j*2*pi/params.l*(np.outer(params.xq[:,0],u) + np.outer(params.xq[:,1],v)))/np.sqrt(Nsensor)#))/np.sqrt(Nsensor)
A = A/np.linalg.norm(A, axis=0)

print('Starting angles: ' ,theta[0,0], theta[1,0], theta[2,0], theta[3,0], theta[4,0])

## make data
TEST = np.zeros((params.Nsensor, snap, nsamples), dtype=complex)
csdm = np.zeros((nsamples, params.Nsensor, params.Nsensor),dtype=complex)
x_test = np.zeros((nsamples, Nfeat))
y_test = np.zeros((nsamples, params.nclasses))

for i_theta in np.arange(nsamples):
        randseed( seed[i_theta] )

        y, Xsource = generate_signal(theta[:,i_theta], Amp, A, snr, snap, params)
        TEST[:,:,i_theta] = y
        csdm[i_theta,:,:] = np.inner(y[:,0:snap], y[:,0:snap].conj())/snap
        x_test[i_theta,:] = form_features(csdm[i_theta,:,:])
        y_test[i_theta,:] = np.squeeze(Xsource[0,:])
labels = theta.T

## Save
if not os.path.isdir(params.fpath + 'test/' + params.fname):
    if not os.path.isdir('data/'):
        os.mkdir('data/')
    if not os.path.isdir(params.fpath):
        os.mkdir(params.fpath)
    if not os.path.isdir(params.fpath + 'test/'):
        os.mkdir(params.fpath + 'test/')
    os.mkdir(params.fpath + 'test/' + params.fname)

## FOR FNN # #
np.save(params.fpath + 'test/' + params.fname + 'xtest' + filename, x_test)
np.save(params.fpath + 'test/' + params.fname + 'ytest' + filename, y_test)
print(params.fpath + 'test/' + params.fname + 'xtest' + filename)

## FOR CBF # #
np.save(params.fpath + 'test/' + params.fname + 'raw' + filename, TEST)
np.save(params.fpath + 'test/' + params.fname + 'labels' + filename, labels)

## FOR SBL # #
data = dict();
data['TEST'] = TEST
data['labels'] = labels
sio.savemat(params.fpath +  'test/' + params.fname + filename[1:], mdict={'data':data})


from kafka.admin import KafkaAdminClient, NewTopic

admin_client = KafkaAdminClient(
    bootstrap_servers="localhost:9092",
    client_id='test'
)

topic_list = []
topic_list.append(NewTopic(name="poc-topic", num_partitions=2, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)

# Use this to delete the topic
#admin_client.delete_topics(['poc-topic'], timeout_ms=10000)


#!/usr/bin/env python3
"""
Author      : Madhu Babu Kencha
Created on  : 14-sept-2018

1) In this program we will see how to combain both positional and
   optional arguments
2) Order of passing arguments does not matter
"""

import argparse

def main():
	parser = argparse.ArgumentParser()
	parser.add_argument("name", help="Print you the name")
	parser.add_argument("-v", "--verbose", help="Print the verbose output",
		                action="store_true")
	args = parser.parse_args()
	your_name = args.name
	if args.verbose:
		print("The programmer name is {}".format(your_name))
	else:
		print(your_name)

if __name__ == '__main__':
	main()


from random import randint, uniform, choice
from pygame import time
from lib.object.game.Axis import Axis
from lib.object.game.Item import Item, get_random_effect
from lib.object.players.PlayerVampire import PlayerVampire
from lib.utils.Utils import Utils

class ItemManager:
    def __init__(self, number_manager):
        super().__init__()
        self.items = []
        self.number_manager = number_manager  # TODO - verificar se é a melhor maneira

    def calculate_rates(self):
        heals = 0
        atk = 0
        speed = 0
        count = len(self.items) + 1

        for item in self.items:
            if item.effect == self.raise_damage:
                atk += 1
            elif item.effect == self.raise_attack_speed:
                speed += 1
            else:
                heals += 1

        print(
            f"atk: {round((atk / count) * 100)}% speed: {round((speed / count) * 100)}% heal: {round((heals / count) * 100)}%")

    def reset(self):
        self.items = []

    def random_item(self, x, y):
        if randint(1, 3) == 1:
            self.create_item(x, y)

    def create_item(self, x, y):
        effect = get_random_effect(self)
        new_item = Item(
            x=x,
            y=y,
            speed=Axis(uniform(-2, 2), randint(1, 3)),
            sprite=Utils.scale_image(effect['sprite'], 0.3),
            effect=effect['effect'],
        )
        new_item.set_size_with_sprite()
        new_item.center()
        self.items.append(new_item)

    def move_item(self, item, render_frame_time):
        item.x += item.speed.x * render_frame_time
        item.y += item.speed.y * render_frame_time

    def has_collided(self, item, object, *actions):
        if item.collided_with(object):
            try:
                for action in actions:
                    action(item)
            except:
                print("error in item collision action")

    def check_item(self, item, screen_size):
        if item.y < -(item.size.y * 2):
            self.items.remove(item)
        elif item.x < -(item.size.x * 2):
            self.items.remove(item)
        elif item.x > screen_size.x:
            self.items.remove(item)
        elif item.y > screen_size.y:
            self.items.remove(item)

    def heal(self, player):
        heal_value = uniform(player.max_health * 0.1, player.max_health * 0.15)
        if type(player) is not PlayerVampire:
            if player.health <= player.max_health:
                if heal_value + player.health >= player.max_health:
                    player.health = player.max_health
                    text = 'MAX'
                else:
                    player.health += heal_value
                    text = f'+{round(heal_value)}'

                self.number_manager.add_heal_number(player.x,
                                                    player.y, text)
        else:
            player.health -= player.max_health * 0.5
            self.number_manager.add_take_damage_number(player.x, player.y, player.max_health * 0.5)

    def raise_attack_speed(self, player):
        atk_speed = player.weapon.shoot_delay * uniform(0.93, 0.97)
        if atk_speed > 0:
            player.weapon.shoot_delay = atk_speed
        else:
            player.weapon.shoot_delay = 0

        self.number_manager.add_buff_info(player.x, player.y, "+F. RATE")

    def raise_damage(self, player):
        damage = player.weapon.bullet.damage
        player.weapon.bullet.damage += uniform(damage * 0.1, damage * 0.15)
        self.number_manager.add_buff_info(player.x, player.y, "+ATK")

    def raise_move_speed(self, player):
        player.speed.x += uniform(0.7, 2.2)
        player.speed.y += uniform(0.3, 1.5)

    def change_weapon_type(self, player):
        types = ["single", "double", "triple", "spread", "wiggle", "arc"]
        types.remove(player.weapon.weapon_type)
        player.weapon.weapon_type = choice(types)

    def recharge_ultimage(self, player):
        player.next_ult = time.get_ticks()
        self.number_manager.add_buff_info(player.x, player.y, "+ULT")

    def raise_crit_rate(self, player):
        player.crit_rate+=0.07
        self.number_manager.add_buff_info(player.x, player.y, "+CRIT")


import pandas as pd

# Loading Dataset of Molecules Solubility using Physics-informed Learning
df = pd.read_csv('https://raw.githubusercontent.com/oplogix/Projects/main/First-ML-Projects/delaney_solubility_with_descriptors.csv')

#Data Preparation

#Data separation as X and Y

y = df['logS']

x = df.drop('logS', axis=1)   #Axis: 1 = Column, 0 = Row

#Data Splitting

from sklearn.model_selection import train_test_split  # From scikit-learn module

#Training set = 80% of the data, while the test set = 20% of the data.
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=100) # Defining new variables

# Linear Regression Model - First Model

# Training the Model
from sklearn.linear_model import LinearRegression
lr = LinearRegression()
# Train LR Model on this dataset
lr.fit(x_train, y_train)

LinearRegression()

# Applying the model to make a prediction
y_lr_train_pred = lr.predict(x_train)
y_lr_test_pred = lr.predict(x_test)

# Evaluate Model Performance
from sklearn.metrics import mean_squared_error, r2_score

lr_train_mse = mean_squared_error(y_train, y_lr_train_pred)
lr_train_r2 = r2_score(y_train, y_lr_train_pred)

lr_test_mse = mean_squared_error(y_test, y_lr_test_pred)
lr_test_r2 = r2_score(y_test, y_lr_test_pred)

# Organizing Data of LR Model
lr_results = pd.DataFrame(['Linear Regression', lr_train_mse, lr_train_r2, lr_test_mse, lr_test_r2]).transpose()
lr_results.columns = ['Method', 'Training MSE', 'Training R2', 'Test MSE', 'Test R2']

# Random Forest Training Model - Second model

# Training the Model
from sklearn.ensemble import RandomForestRegressor   # Using Regressor because Y value (LogS in Quantatative), Categorical would be Classification model

rf = RandomForestRegressor(max_depth=2, random_state=100)
rf.fit(x_train, y_train)

# Applying the Model to make a Prediction
y_rf_train_pred = rf.predict(x_train)
y_rf_test_pred = rf.predict(x_test)

# Evaluating Models Performance

rf_train_mse = mean_squared_error(y_train, y_rf_train_pred)
rf_train_r2 = r2_score(y_train, y_rf_train_pred)

rf_test_mse = mean_squared_error(y_test, y_rf_test_pred)
rf_test_r2 = r2_score(y_test, y_rf_test_pred)

# Organizing RF Data

rf_results = pd.DataFrame(['Random Forest', rf_train_mse, rf_train_r2, rf_test_mse, rf_test_r2]).transpose()
rf_results.columns = ['Method', 'Training MSE', 'Training R2', 'Test MSE', 'Test R2']

# Model Comparison
df_models = pd.concat([lr_results, rf_results], axis=0).reset_index(drop=True)
print(df_models)

# Creating ScatterPlot

import matplotlib.pyplot as plt
import numpy as np

x = [y_train]
y = [y_lr_train_pred]

plt.figure(figsize=(10,10))
plt.scatter(x, y, alpha=0.3)
plt.title("Linear Regression Graph")
plt.ylabel('Predict LogS')
plt.xlabel('Expirimental LogS')
plt.show()

# #Get Trendline Coefficients and Polynomial
# z = np.polyfit(x, y, 1) -  ("expected 1D vector for x") - https://stackoverflow.com/questions/33177548/typeerror-expected-1d-vector-for-x
# p = np.poly1d(z)

# #Add Trendline
# plt.plot(x, p(x), '#F8766D')


'''Constants used in TemplateSB.'''

import os
import yaml

# The following constants are specified in the config file
COMMAND_START = None
COMMAND_END = None
COMMENT_STG = None
CONTINUED_STG = None
EXPRESSION_START = None
EXPRESSION_END = None
# Number of definitions before a warning is generated
WARNING_ASSIGNMENTS = 10000

# The following constants are internal
PARENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CONFIG_FILE_PATH = os.path.join(PARENT_DIR, "config.yaml")
LINE_COMMAND = 2  # Command line
LINE_NONE = -1  # Command line
LINE_SUBS = 3  # Line to be processed for substitutions
LINE_TRAN = 1  # Transparent - nothing to process (comment line, no template variable)
LINE_NONE = -1 # No more lines
# Version of code
VERSION = "1.2"
# YAML keywords and their internal counterparts
PAIRS = {"command_start": "COMMAND_START",
         "command_end": "COMMAND_END",
         "comment_character": "COMMENT_STG",
         "continuation_string": "CONTINUED_STG",
         "expression_start": "EXPRESSION_START",
         "expression_end": "EXPRESSION_END",
         "warn_assignments": "WARNING_ASSIGNMENTS",
        }
SPLIT_STG = '\n'

def setConstantsFromConfig(path=CONFIG_FILE_PATH):
  fd = open(path, "r")
  lines = ''.join(fd.readlines())
  fd.close()
  namespace = globals()
  config_dict = yaml.load(lines)
  for key, value in config_dict.items():
    if key in PAIRS.keys():
      namespace[PAIRS[key]] = value
    else:
      raise ValueError("Invalid configuration parameter: %s" % key)
  # Check that all values have been assigned
  for name in PAIRS.values():
    if not name in namespace:
    # FIND THE VALUE
      raise ValueError("No configuration parameter: %s" % key)

def getGlobals():
  return globals()

setConstantsFromConfig()


# Written by Haram Choi
import torch
import torch.nn as nn
from einops import rearrange
from einops.layers.torch import Rearrange

class InterPool(nn.Module):
    def __init__(self, dim):
        super(InterPool, self).__init__()
        self.dim = dim
        self.max_pool = nn.MaxPool2d(2)

    def forward(self, x, num_patches):
        x = rearrange(x, 'b (h w) c -> b c h w', h=num_patches[0])
        x = self.max_pool(x)
        x = rearrange(x, 'b c h w -> b (h w) c')
        return x

    def flops(self, resolutions):
        H, W, D = resolutions
        flops = 0
        flops += (H//2)*(W//2) * 2*2 * D # max_pool
        return flops

class BottleneckPool(nn.Module):
    def __init__(self, dim):
        super(BottleneckPool, self).__init__()
        self.dim = dim

        self.max_pool = nn.MaxPool2d(2)
        self.act = nn.LeakyReLU()
    def forward(self, x, exp):
        for _ in range(exp):
            x = self.max_pool(x)
        x = self.act(x)
        x = rearrange(x, 'b c h w -> b (h w) c')
        return x

    def flops(self, resolutions_origin, resolutions):
        import math
        Ho, Wo, H, W = resolutions_origin+resolutions
        exp = int(math.log2(Ho//H))
        flops = 0
        Hr, Wr = Ho//2, Wo//2
        for e in range(exp):
            flops += Hr*Wr * 2*2 * self.dim # max_pool
            Hr, Wr = Hr//2, Wr//2
        flops += H*W*self.dim # act
        return flops


# -*- coding: utf-8 -*-
from collections import deque
from index.ma import MA
from math import sqrt
from functools import reduce
from operator import add

class Boll:
    def __init__(self, period=20):
        self.period = period
        self.ma = MA(period)
        self.queue = deque(maxlen=period)
        self.chunk_id = None

    def __call__(self, item):
        identifier, value = item

        ma  = self.ma((identifier, value["close"]))
        if ma:
            if identifier != self.chunk_id:
                self.chunk_id = identifier
                self.queue.append((value["close"] - ma) ** 2)
            else:
                self.queue[-1] = (value["close"] - ma) ** 2
            stdev = sqrt(reduce(add, self.queue) / self.period)
            return {"upper": ma + 2 * stdev,
                    "lower": ma - 2 * stdev,
                    "mid": ma, **value}


r"""Inspect a TFRecord file of tensorflow.Example and show tokenizations.

python data_generators/inspect_tfrecord.py \
    --logtostderr \
    --print_targets \
    --subword_text_encoder_filename=$DATA_DIR/vocab.endefr.8192 \
    --input_filename=$DATA_DIR/wmt_ende_tokens_8k-train-00000-of-00100
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import six

from tensor2tensor.data_generators import text_encoder

import tensorflow.compat.v1 as tf

tf.flags.DEFINE_string("subword_text_encoder_filename", "",
                       "SubwordTextEncoder vocabulary file")
tf.flags.DEFINE_string("token_text_encoder_filename", "",
                       "TokenTextEncoder vocabulary file")
tf.flags.DEFINE_bool("byte_text_encoder", False, "use a ByteTextEncoder")
tf.flags.DEFINE_string("input_filename", "", "input filename")
tf.flags.DEFINE_bool("print_inputs", False, "Print decoded inputs to stdout")
tf.flags.DEFINE_bool("print_targets", False, "Print decoded targets to stdout")
tf.flags.DEFINE_bool("print_all", False, "Print all fields")

FLAGS = tf.flags.FLAGS

def main(_):
  """Convert a file to examples."""
  if FLAGS.subword_text_encoder_filename:
    encoder = text_encoder.SubwordTextEncoder(
        FLAGS.subword_text_encoder_filename)
  elif FLAGS.token_text_encoder_filename:
    encoder = text_encoder.TokenTextEncoder(FLAGS.token_text_encoder_filename)
  elif FLAGS.byte_text_encoder:
    encoder = text_encoder.ByteTextEncoder()
  else:
    encoder = None
  reader = tf.python_io.tf_record_iterator(FLAGS.input_filename)
  total_sequences = 0
  total_input_tokens = 0
  total_target_tokens = 0
  nonpadding_input_tokens = 0
  nonpadding_target_tokens = 0
  max_input_length = 0
  max_target_length = 0
  for record in reader:
    x = tf.train.Example()
    x.ParseFromString(record)
    inputs = [int(i) for i in x.features.feature["inputs"].int64_list.value]
    targets = [int(i) for i in x.features.feature["targets"].int64_list.value]
    if FLAGS.print_inputs:
      print("INPUTS:\n" + encoder.decode(inputs) if encoder else inputs)
    if FLAGS.print_targets:
      print("TARGETS:\n" + encoder.decode(targets) if encoder else targets)
    nonpadding_input_tokens += len(inputs) - inputs.count(0)
    nonpadding_target_tokens += len(targets) - targets.count(0)
    total_input_tokens += len(inputs)
    total_target_tokens += len(targets)
    total_sequences += 1
    max_input_length = max(max_input_length, len(inputs))
    max_target_length = max(max_target_length, len(targets))
    if FLAGS.print_all:
      for k, v in six.iteritems(x.features.feature):
        print("%s: %s" % (k, v.int64_list.value))

  print("total_sequences: %d" % total_sequences)
  print("total_input_tokens: %d" % total_input_tokens)
  print("total_target_tokens: %d" % total_target_tokens)
  print("nonpadding_input_tokens: %d" % nonpadding_input_tokens)
  print("nonpadding_target_tokens: %d" % nonpadding_target_tokens)
  print("max_input_length: %d" % max_input_length)
  print("max_target_length: %d" % max_target_length)

if __name__ == "__main__":
  tf.app.run()


#!/usr/bin/env python3
import sys
import unittest
sys.path.append('./fizzbuzzpy')
from src.fizzbuzz import fizzbuzz

FAILURE = 'incorrect value'

class FizzbuzzTest(unittest.TestCase):
    def test_returns_fizz_for_multiple_of_3(self):
        value = fizzbuzz(3)
        value2 = fizzbuzz(9)
        self.assertEqual(value, 'fizz', FAILURE)
        self.assertEqual(value2, 'fizz', FAILURE)

    def test_returns_fizz_for_multiple_of_5(self):
        value = fizzbuzz(5)
        value2 = fizzbuzz(25)
        self.assertEqual(value, 'buzz', FAILURE)
        self.assertEqual(value2, 'buzz', FAILURE)

    def test_returns_fizzbuzz_for_multiple_of_3_and_5(self):
        value = fizzbuzz(15)
        value2 = fizzbuzz(75)
        self.assertEqual(value, 'fizzbuzz', FAILURE)
        self.assertEqual(value2, 'fizzbuzz', FAILURE)

    def test_returns_zero_for_zero(self):
        value = fizzbuzz(0)
        self.assertEqual(value, '0', FAILURE)

    def test_returns_original_number(self):
        value = fizzbuzz(1)
        value2 = fizzbuzz(74)
        self.assertEqual(value, '1', FAILURE)
        self.assertEqual(value2, '74', FAILURE)

    def test_returns_invalid_for_negative_numbers(self):
        value = fizzbuzz(-1)
        value2 = fizzbuzz(-74)
        self.assertEqual(value, 'invalid number', FAILURE)
        self.assertEqual(value2, 'invalid number', FAILURE)

if __name__ == '__main__':
    unittest.main()


import sieve
from typing import Dict, Tuple
from yolo import Yolo
from tracker import SORT
from visualizer import draw_boxes

metadata = sieve.Metadata(
    title="Track Objects in Video",
    description="Run hyperparallelized object tracking on video.",
    code_url="https://github.com/sieve-community/examples/tree/main/yolo_object_tracking/main.py",
    image=sieve.Image(
        url="https://storage.googleapis.com/sieve-public-data/yolo_object_tracking/cover.webp"
    ),
    tags=["Tracking", "Video", "Detection"],
    readme=open("README.md", "r").read(),
)

@sieve.workflow(name="object_tracking", metadata=metadata)
def yolosplit(video: sieve.Video) -> Dict:
    """
    :param video: Video of objects to be tracked
    :return: Dictionary of tracked objects
    """
    images = sieve.reference("sieve/video-splitter")(video)
    yolo_outputs = Yolo()(images)
    return SORT(yolo_outputs)

@sieve.workflow(name="object_tracking_visualize", metadata=metadata)
def yolo_visualize(video: sieve.Video) -> Tuple[sieve.Video, Dict]:
    """
    :param video: Video of objects to be tracked
    :return: Original video with tracked objects overlayed and a dictionary of tracked objects
    """

    images = sieve.reference("sieve/video-splitter")(video)
    yolo_outputs = Yolo()(images)
    visualized = draw_boxes(images, yolo_outputs)
    combined = sieve.reference("sieve/frame-combiner")(visualized)
    return combined, SORT(yolo_outputs)

if __name__ == "__main__":
    sieve.push(
        yolosplit,
        inputs={
            "video": {
                "url": "https://storage.googleapis.com/sieve-public-videos-grapefruit/bike.mp4"
            }
        },
    )