from __future__ import annotations

__all__ = [
    "check_jax",
    "check_numpy",
    "check_pandas",
    "check_polars",
    "check_torch",
    "check_xarray",
    "is_jax_available",
    "is_numpy_available",
    "is_pandas_available",
    "is_polars_available",
    "is_torch_available",
    "is_xarray_available",
]

from importlib.util import find_spec

def is_jax_available() -> bool:
    r"""Indicates if the ``jax`` package is installed or not.

    Returns:
    -------
        bool: ``True`` if ``jax`` is available otherwise ``False``.
    """
    return find_spec("jax") is not None

def check_jax() -> None:
    r"""Checks if the ``jax`` package is installed.

    Raises:
    ------
        RuntimeError if the ``jax`` package is not installed.
    """
    if not is_jax_available():
        raise RuntimeError(
            "`jax` package is required but not installed. "
            "You can install `jax` package with the command:\n\n"
            "pip install jax\n"
        )

def is_numpy_available() -> bool:
    r"""Indicates if the ``numpy`` package is installed or not.

    Returns:
    -------
        bool: ``True`` if ``numpy`` is available otherwise ``False``.
    """
    return find_spec("numpy") is not None

def check_numpy() -> None:
    r"""Checks if the ``numpy`` package is installed.

    Raises:
    ------
        RuntimeError if the ``numpy`` package is not installed.
    """
    if not is_numpy_available():
        raise RuntimeError(
            "`numpy` package is required but not installed. "
            "You can install `numpy` package with the command:\n\n"
            "pip install numpy\n"
        )

def is_pandas_available() -> bool:
    r"""Indicates if the ``pandas`` package is installed or not.

    Returns:
    -------
        bool: ``True`` if ``pandas`` is available otherwise ``False``.
    """
    return find_spec("pandas") is not None

def check_pandas() -> None:
    r"""Checks if the ``pandas`` package is installed.

    Raises:
    ------
        RuntimeError if the ``pandas`` package is not installed.
    """
    if not is_pandas_available():
        raise RuntimeError(
            "`pandas` package is required but not installed. "
            "You can install `pandas` package with the command:\n\n"
            "pip install pandas\n"
        )

def is_polars_available() -> bool:
    r"""Indicates if the ``polars`` package is installed or not.

    Returns:
    -------
        bool: ``True`` if ``polars`` is available otherwise ``False``.
    """
    return find_spec("polars") is not None

def check_polars() -> None:
    r"""Checks if the ``polars`` package is installed.

    Raises:
    ------
        RuntimeError if the ``polars`` package is not installed.
    """
    if not is_polars_available():
        raise RuntimeError(
            "`polars` package is required but not installed. "
            "You can install `polars` package with the command:\n\n"
            "pip install polars\n"
        )

def is_torch_available() -> bool:
    r"""Indicates if the ``torch`` package is installed or not.

    Returns:
    -------
        bool: ``True`` if ``torch`` is available otherwise ``False``.
    """
    return find_spec("torch") is not None

def check_torch() -> None:
    r"""Checks if the ``torch`` package is installed.

    Raises:
        RuntimeError if the ``torch`` package is not installed.
    """
    if not is_torch_available():
        raise RuntimeError(
            "`torch` package is required but not installed. "
            "You can install `torch` package with the command:\n\n"
            "pip install torch\n"
        )

def is_xarray_available() -> bool:
    r"""Indicates if the ``xarray`` package is installed or not.

    Returns:
    -------
        bool: ``True`` if ``xarray`` is available otherwise ``False``.
    """
    return find_spec("xarray") is not None

def check_xarray() -> None:
    r"""Checks if the ``xarray`` package is installed.

    Raises:
    ------
        RuntimeError if the ``xarray`` package is not installed.
    """
    if not is_xarray_available():
        raise RuntimeError(
            "`xarray` package is required but not installed. "
            "You can install `xarray` package with the command:\n\n"
            "pip install xarray\n"
        )


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from appium import webdriver

import desired_capabilities

import unittest
from time import sleep
import json

# the emulator is sometimes slow and needs time to think
SLEEPY_TIME = 1

class AppiumTests(unittest.TestCase):
    def setUp(self):
        desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk')
        self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps)

    def tearDown(self):
        self.driver.quit()

    def test_app_strings(self):
        strings = self.driver.app_strings
        self.assertEqual(u'You can\'t wipe my data, you are a monkey!', strings[u'monkey_wipe_data'])

    def test_keyevent(self):
        # not sure how to test this.
        self.driver.keyevent(176)

    def test_current_activity(self):
        activity = self.driver.current_activity
        self.assertEqual('.ApiDemos', activity)

    def test_set_value(self):
        el = self.driver.find_element_by_class_name('android.widget.ListView')
        js_params = {'element': el.id, 'text': "Views"}
        self.driver.execute_script("mobile: scrollTo", js_params)

        el = self.driver.find_element_by_name('Views')
        el.click()

        el = self.driver.find_element_by_name('Auto Complete')
        el.click()

        el = self.driver.find_element_by_name('4. Contacts')
        el.click()

        el = self.driver.find_element_by_class_name('android.widget.EditText')
        self.driver.set_value(el, 'Isaac')

        text = el.get_attribute('text')
        self.assertEqual('Isaac', text)

    def test_element_set_value(self):
        el = self.driver.find_element_by_class_name('android.widget.ListView')
        js_params = {'element': el.id, 'text': "Views"}
        self.driver.execute_script("mobile: scrollTo", js_params)

        el = self.driver.find_element_by_name('Views')
        el.click()

        el = self.driver.find_element_by_name('Auto Complete')
        el.click()

        el = self.driver.find_element_by_name('4. Contacts')
        el.click()
        sleep(SLEEPY_TIME)

        el = self.driver.find_element_by_class_name('android.widget.EditText')
        el.set_value('Isaac')

        text = el.get_attribute('text')
        self.assertEqual('Isaac', text)

    def test_pull_file(self):
        data = self.driver.pull_file('data/local/tmp/strings.json')
        strings = json.loads(data.decode('base64', 'strict'))
        self.assertEqual('You can\'t wipe my data, you are a monkey!', strings[u'monkey_wipe_data'])

    def test_push_file(self):
        path = 'data/local/tmp/test_push_file.txt'
        data = 'This is the contents of the file to push to the device.'
        self.driver.push_file(path, data.encode('base64'))
        data_ret = self.driver.pull_file('data/local/tmp/test_push_file.txt').decode('base64')
        self.assertEqual(data, data_ret)

    def test_complex_find(self):
        # this only works with a three dimensional array like here.
        el = self.driver.complex_find([[[2, 'Ani']]])
        self.assertIsNotNone(el)

    def test_background_app(self):
        self.driver.background_app(1)
        sleep(5)
        el = self.driver.find_element_by_name('Animation')
        self.assertIsNotNone(el)

    def test_is_app_installed(self):
        self.assertFalse(self.driver.is_app_installed('sdfsdf'))
        self.assertTrue(self.driver.is_app_installed('com.example.android.apis'))

    # # this causes the server to crash. no idea why
    # def test_install_app(self):
    #     self.assertFalse(self.driver.is_app_installed('io.selendroid.testapp'))
    #     self.driver.install_app('/Users/isaac/code/python-client/test/apps/selendroid-test-app.apk')
    #     self.assertTrue(self.driver.is_app_installed('io.selendroid.testapp'))

    def test_remove_app(self):
        self.assertTrue(self.driver.is_app_installed('com.example.android.apis'))
        self.driver.remove_app('com.example.android.apis')
        self.assertFalse(self.driver.is_app_installed('com.example.android.apis'))

    def test_close__and_launch_app(self):
        el = self.driver.find_element_by_name('Animation')
        self.assertIsNotNone(el)

        self.driver.close_app();

        self.driver.launch_app()

        el = self.driver.find_element_by_name('Animation')
        self.assertIsNotNone(el)

    # # not sure how to set this up to run
    # def test_end_test_coverage(self):
    #     self.driver.end_test_coverage(intent='android.intent.action.MAIN', path='')
    #     sleep(5)

    def test_reset(self):
        el = self.driver.find_element_by_name('App')
        el.click()

        self.driver.reset()
        sleep(5)

        el = self.driver.find_element_by_name('App')
        self.assertIsNotNone(el)

if __name__ == "__main__":
    unittest.main()


# The goal of this script is to get all the stats from the sensors to:
# - print to the epaper screen
# - print to plant_stats.csv - you can then use a spreadsheet of choice and draw cool graphs from it.
# - Drive the different microcontrolers to control inside climate: turn on/off fans, heat mat (not done), and water the plan if needed.
# (this last one does overlap with the auto_water.py... you choose which one to use)
# I put it in crontab to start a reboot
# $ sudo crontab -l
# @reboot cd /home/pi/Documents/SpaceBucket/FlaskWateringServer/; python3 print_status_epaper.py

import water
import epaper
import datetime
import time
import RPi.GPIO as GPIO

if __name__ == "__main__":

    # To use on the while loop
    booleankey = True

    PUMP_PIN=38 # To relay channel IN1
    IN2_PIN=36 # for fans
    IN3_PIN=32 # available... potentially heat mat
    IN4_PIN=37 # for lights

    # Variables used to control refresh and automation
    delaytime = 600
    temp_low_limit = 20.0
    temp_high_limit = 25.0
    hum_low_limit = 50.0
    hum_high_limit = 75.0
    dry_days_limit = 4
    soil_moist_low_limit = 25

    while (booleankey):
        try:
            string[0] = "{}".format(datetime.datetime.now().strftime("%a %d %b %Y %I:%M%p"))

            soil_status = water.get_status()
            soil_moist_percent = water.get_soil_moist_percent()
            days_since_watered = water.get_days_since_watered()
            temp, temp_F, hum = water.get_tuple_temp_hum()

            if (soil_status == 1):
                string[1] = "Soil is DRY!"

            else:
                string[1] = "Soil is WET"

            string[2] = "{0:0.1f} days since water".format(days_since_watered)
            string[3] = "Soil Moisture: {0:0.1f}%".format(soil_moist_percent)
            string[4] = "Temp: {0:0.1f}*C/{1:0.1f}*F".format(temp,temp_F)
            string[5] = "Humidity: {0:0.1f}%".format(hum)

            #print all 4 lines to epaper
            epaper.write_epaper(string[0],string[1],string[2],string[3],string[4],string[5])

            #print to stats to .csv file as well (timestamp, temp, tempF, hum, soilmoisture, fan on?, light on?, days since watered)
            is_fan_on= water.check_gpio_status(IN2_PIN)
            is_light_on=water.check_gpio_status(IN4_PIN)

            now_isoformat = datetime.datetime.now().isoformat()

            # Decided to put the stats file in ./static to be able to easily access it via a web browser
            f = open('./static/plant_stats.csv', "a")
            f.write("{0},{1:0.1f},{2:0.1f},{3:0.1f},{4:0.1f},{5},{6},{7}\n".format(now_isoformat,temp,temp_F,hum,soil_moist_percent,is_fan_on,is_light_on,days_since_watered))
            f.close()

            # Low temperature, low RH, then turn fans off
            if (temp < temp_low_limit or hum < hum_low_limit):
                water.fans_off()
            # high temperature, high humidity, then turn fans on
            elif (temp > temp_high_limit or hum > hum_high_limit):
                water.fans_on()

            # If soil moisture is too low (dry), then turn on pump for default 5min
            if (days_since_watered > dry_days_limit):
                water.pump_on_demand()

            elif (soil_moist_percent < soil_moist_low_limit):
                water.pump_on_demand()

            time.sleep(delaytime)

        except KeyboardInterrupt:
            booleankey = False


import sys
import argparse

from taggers import stub
from taggers import corenlp
from taggers import nltkchunker
from taggers import spacy

# Download from https://nlp.stanford.edu/software/CRF-NER.html#Download
CORENLP_DIR = '../../stanford-ner-2018-02-27'

def make_tagger(name):
    if name == 'stub':
        return stub.StubNer()
    elif name == 'corenlp':
        return corenlp.StanfordNer(corenlp_dir=CORENLP_DIR)
    elif name == 'nltkchunker':
    	return nltkchunker.ChunkerNer()
    elif name == 'spacy':
        return spacy.SpacyNer()
    else:
        print('ERROR: Unrecognized tagger:', name)
        sys.exit(1)

if __name__ == '__main__':
    argparser = argparse.ArgumentParser(description='Find Named Entities in a sentence')
    argparser.add_argument('sentence', metavar='SENTENCE', type=str)
    argparser.add_argument('-t, --tagger', dest='tagger', type=str, default='stub')

    args = argparser.parse_args()

    ner = make_tagger(args.tagger)
    result = ner.predict(args.sentence)

    for w, t in result:
        print('{}\t{}'.format(w, t))


from urllib.parse import parse_qs
from html import escape
import db
import http_wsgi as hw
import os
from string import Template
from hashlib import md5

TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), 'templates')

def render(template_name):
    template_path = os.path.join(TEMPLATE_DIR, template_name)
    mass_hash = db.select_hash_db()
    with open(template_path, 'r') as f:
        response_body = f.read()
        response_body += '
hash in data base

'
        for h in mass_hash:
            response_body += h
        response_body += '
'
    template = Template(response_body).safe_substitute().encode()
    return template

def take_hash(url):
    url_hash = md5(url.encode('utf-8')).hexdigest()
    return url_hash[:8]

def take_url_post(environ):
    try:
        size = int(environ.get('CONTENT_LENGTH', 0))
    except ValueError:
        size = 0
    data = environ['wsgi.input'].read(size)
    m_data = parse_qs(data.decode())
    url = m_data['url'][0]
    url_hash = take_hash(url)
    db.add_db_url(url_hash, url)
    print('{} - {}'.format(url_hash, url))

def app(environ, start_response):
    template = render('index.html')
    if environ['REQUEST_METHOD'] == 'POST':
        take_url_post(environ)
    elif environ['REQUEST_METHOD'] == 'GET':
        path_inf = environ.get('PATH_INFO', '')
        path = path_inf.split('/')
        qs = parse_qs(environ['QUERY_STRING'])
        if path[1] != '':
            try:
                return hw.redirect_url(environ, start_response, path[1])
            except TypeError:
                return hw.not_found(environ, start_response)
        elif len(qs) > 0:
            try:
                url_hash = escape(qs.get('hash', [''])[0])
                return hw.redirect_url(environ, start_response, url_hash)
            except TypeError:
                return hw.not_found(environ, start_response)
    return hw.index_html(environ, start_response, template)

if __name__ == '__main__':
    try:
        from wsgiref.simple_server import make_server
        http_srv = make_server('localhost', 8080, app)
        http_srv.serve_forever()
    except KeyboardInterrupt:
        print('Goodbye.')


from args import parse_config
from utils import *
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np
import torch
from VAE_trainer import VAE_trainer
from AE_trainer import AE_trainer
from BFM_trainer import BFM_trainer
from RBSVD_trainer import RBSVD_trainer
from IterativeSVD_trainer import IterativeSVD_trainer
from SVD_trainer import SVD_trainer
from ALS_trainer import ALS_trainer
from NCF_trainer import NCF_trainer
import os

def get_trainer(**args):
    '''
    Trainer function wrapper. One of ['ae', 'vae', 'bfm', 'iterative_svd', 'rbsvd', 'svd', 'als', 'ncf'].
    '''
    model_type = args.get('model_type')
    if model_type == 'ae':
        return AE_trainer(**args)
    elif model_type == 'vae':
        return VAE_trainer(**args)
    elif model_type == 'bfm':
        return BFM_trainer(**args)
    elif model_type == 'rbsvd':
        return RBSVD_trainer(**args)
    elif model_type == 'iterative_svd':
        return IterativeSVD_trainer(**args)
    elif model_type == 'svd':
        return SVD_trainer(**args)
    elif model_type == 'als':
        return ALS_trainer(**args)
    elif model_type == 'ncf':
        return NCF_trainer(**args)

def train(**args):
    df_all = pd.read_csv(args.get('train_csv_path'))
    final_model = args.get('final_model')
    if not os.path.exists(args.get('ckpt_folder')):
        os.makedirs(args.get('ckpt_folder'))
    if not os.path.exists(args.get('pred_folder')):
        os.makedirs(args.get('pred_folder'))

    if final_model:
        # Training final model using all available data, loading the public test set as df_test (without ground truth).
        df_train = df_all
        df_test = pd.read_csv(args.get('test_csv_path'))
    else:
        # Split the data into train and test set
        df_train, df_test = train_test_split(df_all, test_size=args.get('test_size'), random_state=args.get('random_seed'))
    # Extract user, movie, rating vectors from the dataframe.
    users_train, movies_train, ratings_train = extract_user_movie_rating_arrays(df_train)
    users_test, movies_test, ratings_test = extract_user_movie_rating_arrays(df_test)

    print('Number of training sumples: {}, number of test samples: {}.'.format(len(users_train), len(users_test)))

    trainer = get_trainer(**args)
    trainer.train(users_train, movies_train, ratings_train, users_test, movies_test, ratings_test, **args)

if __name__ == '__main__':
    args = parse_config()
    train(**args)


# -*- coding: utf-8 -*-
from requests_oauthlib import OAuth1Session
import json
import datetime, time, sys, calendar
import os
import codecs
import csv
import urllib.parse
import pandas as pd
from abc import ABCMeta, abstractmethod
from sub.concat_csv import CSV_CONCAT
from sub.shaping_for_fasttext import SHAPING_FOR_FASTTEXT
from sub.pwd import *

base_path = os.path.dirname(os.path.abspath(__file__))
get_tweets_dir = os.path.normpath(os.path.join(base_path, './get_tweets/'))

#tweetをgetしまくる関数。親クラスです。
class TweetsGetter(object):
    __metaclass__ = ABCMeta

    def __init__(self):
        self.session = OAuth1Session(CK, CS, AT, AS)

    #子クラスによるオーバーライドを前提とした関数 abstractmethod
    @abstractmethod
    def specifyUrlAndParams(self, keyword):
        '''
        呼出し先 URL、パラメータを返す
        '''
    #子クラスによるオーバーライドを前提とした関数 abstractmethod
    @abstractmethod
    def pickupTweet(self, res_text, includeRetweet):
        '''
        res_text からツイートを取り出し、配列にセットして返却
        '''
    #子クラスによるオーバーライドを前提とした関数 abstractmethod
    def getLimitContext(self, res_text):
        '''
        回数制限の情報を取得 （起動時）
        '''

    def collect(self, total = -1, onlyText = False, includeRetweet = False):
        '''
        ツイート取得を開始する
        '''
        #----------------
        # 回数制限を確認
        #----------------
        self.checkLimit()

        #----------------
        # URL、パラメータ
        #----------------
        url, params = self.specifyUrlAndParams()
        params['include_rts'] = str(includeRetweet).lower()
        # include_rts は statuses/user_timeline のパラメータ。search/tweets には無効

        #----------------
        # ツイート取得
        #----------------
        cnt = 0
        unavailableCnt = 0
        while True:
            res = self.session.get(url, params = params)
            #api取得エラーだったら以下
            if res.status_code == 503:
                # 503 : Service Unavailable
                if unavailableCnt > 10:
                    #10回以上リピートしてしまったらエラー
                    raise Exception('Twitter API error %d' % res.status_code)
                unavailableCnt += 1
                print ('Service Unavailable 503')
                #30秒待機
                self.waitUntilReset(time.mktime(datetime.datetime.now().timetuple()) + 30)
                continue
            # 503でなければカウントを0にします。
            unavailableCnt = 0

            #api取得エラー以外のガチエラーだったらこちら
            if res.status_code != 200:
                raise Exception('Twitter API error %d' % res.status_code)
            #上記クリアならばtweet用jsonを取得
            tweets = self.pickupTweet(json.loads(res.text))
            #tweetが0ならループから抜けます。
            if len(tweets) == 0:
                # len(tweets) != params['count'] としたいが
                # count は最大値らしいので判定に使えない。
                # ⇒  "== 0" にする
                # https://dev.twitter.com/discussions/7513
                break

            for tweet in tweets:
                #リツイートだったら出力しない。
                if (('retweeted_status' in tweet) and (includeRetweet is False)):
                    pass
                else:
                    #textオプションが入ってたらtextを出力
                    if onlyText is True:
                        yield tweet['text']
                    else:
                        yield tweet

                    cnt += 1
                    # 100件のリクエストごとに件数を表示
                    if cnt % 100 == 0:
                        print ('%d件 ' % cnt)
                    # totalを超えたら出力せずforを抜け出す。
                    if total > 0 and cnt >= total:
                        return
            #以下のmax_id以下ツイートから取得していく。
            #http://nonbiri-tereka.hatenablog.com/entry/2014/03/06/220015
            params['max_id'] = tweet['id'] - 1

            # ヘッダ確認 （回数制限）
            # X-Rate-Limit-Remaining が入ってないことが稀にあるのでチェック
            if ('X-Rate-Limit-Remaining' in res.headers and 'X-Rate-Limit-Reset' in res.headers):
                if (int(res.headers['X-Rate-Limit-Remaining']) == 0):
                    #残り回数が0だったら次の制限解除時間まで待機させる
                    self.waitUntilReset(int(res.headers['X-Rate-Limit-Reset']))
                    #念の為残回数チェック
                    self.checkLimit()
            else:
                #上記のヘッダーが取れなかったらとりあえず残回数を取得
                print ('not found  -  X-Rate-Limit-Remaining or X-Rate-Limit-Reset')
                # 残回数チェック。上記のifで拾えなかった場合こちらで残回数0の場合にリミットまで待機させる。
                self.checkLimit()

    #残回数をチェックし、0だったら待機させる関数
    def checkLimit(self):
        '''
        回数制限を問合せ、アクセス可能になるまで wait する
        '''
        unavailableCnt = 0
        while True:
            url = "https://api.twitter.com/1.1/application/rate_limit_status.json"
            res = self.session.get(url)
            #取れなくなった且つ、残回数取得のapiすら503だったら10回リピートさして、トライさせる
            if res.status_code == 503:
                # 503 : Service Unavailable
                if unavailableCnt > 10:
                    raise Exception('Twitter API error %d' % res.status_code)

                unavailableCnt += 1
                print ('Service Unavailable 503')
                #30秒待機
                self.waitUntilReset(time.mktime(datetime.datetime.now().timetuple()) + 30)
                continue

            unavailableCnt = 0
            # 上記のエラー以外だったら単純にエラーを出力
            if res.status_code != 200:
                raise Exception('Twitter API error %d' % res.status_code)

            #こちらで残り回数と、次までのリセット秒数を拾って、残回数0なら次のリミットまで待機さす
            remaining, reset = self.getLimitContext(json.loads(res.text))
            if (remaining == 0):
                self.waitUntilReset(reset)
            else:
                break

    #残回数をチェックし、0だったら待機させる関数
    def waitUntilReset(self, reset):
        '''
        reset 時刻まで sleep
        '''
        seconds = reset - time.mktime(datetime.datetime.now().timetuple())
        seconds = max(seconds, 0)
        print ('\n     =====================')
        print ('     == waiting %d sec ==' % seconds)
        print ('     =====================')
        #flushは即時実行の関数
        sys.stdout.flush()
        #指定した秒数待機させる
        time.sleep(seconds + 10)  # 念のため + 10 秒

    #staticメソッドにすると、インスタンス化せずともメソッドを呼び出すことができる。
    #そしてclassメソッドと違って最初のself用引数がいらない&ない。 selfがないため、継承して子クラスになっても親クラスのプロパティを参照する。
    @staticmethod
    def bySearch(keyword):
        return TweetsGetterBySearch(keyword)

    @staticmethod
    def byUser(screen_name):
        return TweetsGetterByUser(screen_name)

#上記のクラスの子クラス。サーチ用にゲットする関数やね
class TweetsGetterBySearch(TweetsGetter):
    '''
    キーワードでツイートを検索
    '''
    def __init__(self, keyword):
        #親クラスのinitを呼び出します。 https://www.lifewithpython.com/2014/01/python-super-function.html
        #多分ここで__init__をオーバーライドしているのは self.keywordを作りたかったからでしょう。
        super(TweetsGetterBySearch, self).__init__()
        self.keyword = keyword

    #検索用のパラメータを設定する search tweetsでツイートを取得
    def specifyUrlAndParams(self):
        '''
        呼出し先 URL、パラメータを返す
        '''
        url = 'https://api.twitter.com/1.1/search/tweets.json'
        params = {'q':self.keyword, 'count':200, 'tweet_mode':'extended', 'result_type':'mixed'}
        return url, params

    #collect関数で取得したjson
    def pickupTweet(self, res_text):
        '''
        res_text からツイートを取り出し、配列にセットして返却
        '''
        results = []
        for tweet in res_text['statuses']:
            results.append(tweet)

        return results

    #残回数とリセットまでの時間を取得しreturnするマン
    def getLimitContext(self, res_text):
        '''
        回数制限の情報を取得 （起動時）
        '''
        remaining = res_text['resources']['search']['/search/tweets']['remaining']
        reset     = res_text['resources']['search']['/search/tweets']['reset']
        return int(remaining), int(reset)

def YmdHMS(created_at):
    '''
    create_atの日付を年-月-日 時:日のフォーマットに変換
    '''
    time_utc = time.strptime(created_at, '%a %b %d %H:%M:%S +0000 %Y')
    unix_time = calendar.timegm(time_utc)
    time_local = time.localtime(unix_time)
    return str(time.strftime("%Y-%m-%d %H:%M", time_local))

#そのうちpathを実行スクリプトからの相対パスに変更しようね https://qiita.com/FGtatsuro/items/52ad08640df6bfad5c2a
def dest_tweet(dict, min_faves="0", since_day="2019-01-01"):
    '''
    dictで指定したカテゴリのワードからツイートを拾ってくる。
    '''
    for ct,sw in dict.items():
        since_day = "2018-01-01"
        for kw in sw:
            start_string = """
カテゴリ: {0}
キーワード: {1}
書き込み開始
            """.format(ct,kw)
            print(start_string)
            getter = TweetsGetter.bySearch(u'{0} min_faves:{1} since:{2} lang:ja OR @999999999'.format(kw,min_faves,since_day))
            create_csv_file = get_tweets_dir + "/" + ct + "/" + kw + ".csv"
            file_path_fordir = os.path.dirname(create_csv_file)
            if not os.path.exists(file_path_fordir):
                os.makedirs(file_path_fordir)
            fw = open(create_csv_file, 'w', encoding="utf-8")
            csvwriter = csv.writer(fw)
            write_header = ["ユーザーid","スクリーンネーム","ユーザー名","フォロワー数","リツイート数","ファボ数","投稿日時","ツイート内容"]
            csvwriter.writerow(write_header)
            for tweet in getter.collect(total = 3000):
                write_contents = [
                    str(tweet['user']['id_str']),
                    str(tweet['user']['screen_name']),
                    str(tweet['user']['name']),
                    str(tweet['user']['followers_count']),
                    str(tweet['retweet_count']),
                    str(tweet['favorite_count']),
                    str(YmdHMS(tweet['created_at'])),
                    tweet['full_text']
                    ]
                csvwriter.writerow(write_contents)
            fw.close()
            drop_duplicate(create_csv_file)
            print("書き込み完了\n")

def drop_duplicate(filepath):
    '''
    ユーザーの重複を削除する
    '''
    df = pd.read_csv(filepath)
    df.drop_duplicates(subset=["ユーザーid","ユーザー名"], inplace=True)
    df.to_csv(filepath,index=False)

if __name__ == '__main__':

    serch_dict = {
        "IT":[
            "テスト駆動",
            "講演",
            "dev",
            "参考書",
            "GKE",
            "パターンマッチ",
            "メモリ",
        ],
        "美容":[
            "アクアレーベル",
            "雪肌精",
            "透明肌"
        ],
        "暮らし":[
            "猫",
            "酒場",
            "英国式パブ",
            "一杯",
            "カクテル",
        ],
        "エンタメ":[
            "プラチナメダル",
            "勝った",
            "ワタナベ楽器",
            "ストラップ",
            "ケーブル",
            "ステッカー",
            "アンデッド",
            "パスカット",
            "ドリブル",
            "キャプテン",
            "ユニフォーム",
            "禁忌",
            "猫耳",
            "ティターン",
            "OPTIWEB",
        ]
    }

    # dest_tweet(serch_dict)

    argvs = sys.argv
    is_wakati = argvs[1]
    if is_wakati == "True":
        for k in serch_dict:
            csv_concater = CSV_CONCAT(get_tweets_dir + "/" + k + "/")
            csv_concater.concat_and_dump_csv(get_tweets_dir + "/Integration/label_" + k + ".csv")
        for k in serch_dict:
            shaping_for_fasttext = SHAPING_FOR_FASTTEXT(k, get_tweets_dir + "/Integration/label_" + k + ".csv", get_tweets_dir + "/wakati_txt/")
            shaping_for_fasttext.main()
        shaping_for_fasttext.integrate_txt("Integrate.txt")


import numpy as np, pandas as pd, seaborn as sns, dataframe.datagrid as dg
import os
from sklearn.cluster import KMeans
import scipy.cluster.hierarchy as sch
from scipy.spatial.distance import squareform
from sklearn.utils.testing import all_estimators
from sklearn.preprocessing import LabelBinarizer, StandardScaler, MinMaxScaler
from sklearn.feature_selection import chi2, f_regression, mutual_info_regression, SelectKBest
from sklearn.metrics import mutual_info_score
from sklearn.linear_model import LogisticRegression, Lasso, LinearRegression, Ridge
from sklearn.neural_network import MLPRegressor
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from common_functions import append_dict, partition, df_reorder_columns, timestamp, require_tuple

from dataframe.datasets import DataSet, to_DataSet, hold_dataset, require_extracted
from data.ShapeUp.ShapeUp_data import *
from run_batch import column_filter, run_batch
__author__ = "Clinten Graham"
__maintainer__ = "Clinten Graham"  # 5/20
__email__ = "clintengraham@gmail.com"
__status__ = "Prototype"

def plot_correlation(
        df: pd.DataFrame,
        feature_cnames: list,
        target_cnames: list,
        fontsize=8
    ) -> None:
    '''
    Plot correlation matrix given features and targets, right now there is no distinction between the two, may start
    highlighting target names.

    :param df: dataframe
    :param feature_cnames: list of column names
    :param target_cnames: list of column names
    :param fontsize: Fontsize for labels
    :return: None. Image printed with plt.show()
    '''
    df = df[feature_cnames+target_cnames]
    # Remove Sex from column list, it does not appear in matrix
    if 'DEM_SEX' in df.columns:
        # df['DEM_SEX'].map({'M': 0, 'F': 1}) # did not work with df.corr
        del df['DEM_SEX']

    # Draw heatmap of df
    corr_df = df.dropna().corr()  # make correlation matrix
    g = sns.heatmap(
        corr_df,
        annot=len(feature_cnames+target_cnames) < 10,
        annot_kws={"fontsize":fontsize, "rotation":-45},
        cbar_kws={"orientation": "vertical"}
    )

    # Style options
    g.set_aspect("equal")
    bottom, top = g.get_ylim()
    g.set_ylim(bottom + 0.5, top - 0.5)

    g.set_yticklabels(df.columns, fontsize=fontsize, rotation=0)
    # g.xaxis.set_major_locator(ticker.MultipleLocator(1))
    # x_ticks = list(df.columns)
    # g.set_xticks(np.arange(len(x_ticks)))
    # g.set_xticklabels(x_ticks, fontsize=fontsize, horizontalalignment='center')
    plt.show()
    return corr_df

def feature_explore(df, feature_cname, age_groups=False) -> pd.DataFrame:
    """
    Print histograms of columns in feature_cname and return dataframe of statistics on each column.

    :param df: pandas dataframe
    :param feature_cname: Column name from df (or list of column names)
    :param age_groups: Bool. Describe dataframe over age groups
    :return: dataframe of statistics on each feature_cname
    """
    if isinstance(feature_cname, str):
        feature_cname = [feature_cname]
    # df.hist(by='DEM_SEX',
    #         # sharex=True,
    #         )
    dfm = df.loc[df['DEM_SEX'] == 'M']
    dfm = dfm[feature_cname + ['DEM_AGE']]
    dff = df.loc[df['DEM_SEX'] == 'F']
    dff = dff[feature_cname + ['DEM_AGE']]

    n_feat = len(feature_cname)
    if n_feat == 1:
        # df.hist(column=feature_cname,
        #         by='DEM_SEX',
        #         # sharex=True,
        #         histtype='barstacked',
        #         label=feature_cname
        #         )
        plt.hist([dfm[feature_cname[0]].dropna(), dff[feature_cname[0]].dropna()],
                 histtype='barstacked')
        plt.show()
    else:
        # if n_feat < 6:
        #     f, axes = plt.subplots(2, n_feat, figsize=(3 * 3, 3 * n_feat / 2))
        # else:
        #     # # More rectangular plots for large feature sets
        #     # n_col = math.ceil(n_feat / math.ceil(math.sqrt(len(feature_cname))))
        #     # f, axes = plt.subplots(n_col, 2*n_col, figsize=(6*n_col, 3*n_col))
        n_row = 2*((n_feat // 5) + 1)
        n_col = min(5, n_feat)
        f, axes = plt.subplots(n_row, n_col, figsize=(3 * n_col, 3 * n_row))

        for i, feature in enumerate(feature_cname):
            k = 2 * i
            sns.distplot(dfm[feature].dropna(), color="skyblue", ax=axes[k % n_row, k // n_row])
            k = k + 1
            sns.distplot(dff[feature].dropna(), color="pink", ax=axes[k % n_row, k // n_row])
        plt.show()

    if age_groups:
        # describe by age group if option is True
        age_groups = {**{"18 to 29": [18, 29]},
                      **{str(n) + " to " + str(n + 9): [n, n + 9] for n in range(30, 61, 10)},
                      **{"70 to 90": [70, 90]}}
        dfm_list = []
        dff_list = []
        for group_name, group_range in age_groups.items():
            dfm_list += [dfm.loc[(group_range[0] <= dfm['DEM_AGE']) & (dfm['DEM_AGE'] <= group_range[1])].describe()]
            dff_list += [dff.loc[(group_range[0] <= dff['DEM_AGE']) & (dff['DEM_AGE'] <= group_range[1])].describe()]
        dfm = pd.concat(dfm_list, keys=age_groups.keys())
        dff = pd.concat(dff_list, keys=age_groups.keys())
        df_data = pd.concat([dfm, dff], keys=['M', 'F'])
    else:
        df_data = pd.concat([dfm.describe(), dff.describe()], keys=['M','F'])

    del df_data['DEM_AGE']
    return df_data

def feature_explore_stats(df, feature_cname, age_groups=False) -> pd.DataFrame:
    # feature_explore (stratifies sex and age) but only return feature statistic dataframe (no histograms)
    if isinstance(feature_cname, str):
        feature_cname = [feature_cname]

    dfm = df.loc[df['DEM_SEX'] == 'M']
    dfm = dfm[feature_cname + ['DEM_AGE']]
    dff = df.loc[df['DEM_SEX'] == 'F']
    dff = dff[feature_cname + ['DEM_AGE']]

    if age_groups:
        # describe by age group if option is True
        age_groups = {**{"18 to 29": [18, 29]},
                      **{str(n) + " to " + str(n + 9): [n, n + 9] for n in range(30, 61, 10)},
                      **{"70 to 90": [70, 90]}}
        dfm_list = []
        dff_list = []
        for group_name, group_range in age_groups.items():
            dfm_list += [dfm.loc[(group_range[0] <= dfm['DEM_AGE']) & (dfm['DEM_AGE'] <= group_range[1])].describe()]
            dff_list += [dff.loc[(group_range[0] <= dff['DEM_AGE']) & (dff['DEM_AGE'] <= group_range[1])].describe()]
        dfm = pd.concat(dfm_list, keys=age_groups.keys())
        dff = pd.concat(dff_list, keys=age_groups.keys())
        df_data = pd.concat([dfm, dff], keys=['M', 'F'])
    else:
        df_data = pd.concat([dfm.describe(), dff.describe()], keys=['M','F'])

    del df_data['DEM_AGE']
    return df_data

stats_dict_default = {
    "count": lambda df: np.sum(~np.isnan(df)),
    "mean": np.nanmean,
    "std": np.nanstd,
    "var": np.nanvar,
    "min": np.nanmin,
    "25%": lambda df: np.nanquantile(df, q=.25),
    "50%": lambda df: np.nanquantile(df, q=.5),
    "75%": lambda df: np.nanquantile(df, q=.75),
    "max": np.nanmax,
}
def RefValueTable(df, stats_dict=None) -> pd.DataFrame:
    '''
    Print dataframe of statistics on each column of df from custom dictionary of statistical functions.
    No stratification is performed on df.

    :param df: pd.DataFrame
    :param stats_dict: dictionary with {"func_name": func} key value pairs
    :return: pd.DataFrame
    '''
    if stats_dict is None:
        stats_dict = stats_dict_default
    df = df._get_numeric_data()
    stats_df = pd.DataFrame()
    for stat_name, stat_func in stats_dict.items():
        stats_df[stat_name] = df.apply(stat_func, axis=0)
    return stats_df.T
# print(RefValueTable(df[m_common+m_manual]))
# print(df[m_common+m_manual].describe())  # default describe funtion from pandas

'''
Returns Modified df
'''

def cut_by_IQR(df, q=2, target_cnames = ['BC_DXA_FAT_TOT', 'BC_DXA_LST_TOT']):
    out_list = []
    for target_cname in target_cnames:
        series = df[target_cname].dropna()
        g = sns.boxplot(series)
        plt.show()
        arr = series.to_numpy()
        Q1 = np.quantile(arr, .25)
        Q3 = np.quantile(arr, .75)
        IQR = Q3 - Q1
        BoxMax = Q3 + q * IQR
        outliers = df[['SubjectID', target_cname]].where(series > BoxMax).dropna()
        print(f'Q3 + {q}*IQR = {BoxMax} ({len(outliers)} subjects out of {len(series)})')
        print(outliers['SubjectID'].to_list())
        print(outliers)
        out_list += outliers['SubjectID'].to_list()
    mask = df['SubjectID'].map(lambda x: not(x in out_list))
    return df.loc[mask]

'''
Examples
'''

def corr_cluster(corr_df, thresh=0.5, distance_method='dissimilarity', linkage_method='average', cluster_method='distance'):
    # https://stackoverflow.com/questions/38070478/how-to-do-clustering-using-the-matrix-of-correlation-coefficients
    X = corr_df
    dist = {'pdist': lambda X: sch.distance.pdist(X.values),
            'dissimilarity': lambda X: squareform(1 - np.abs(X))
    }[distance_method]

    hierarchy = sch.linkage(dist(corr_df), method=linkage_method)
    labels = sch.fcluster(hierarchy, thresh * dist(corr_df).max(), cluster_method)
    grouped_columns = [[X.columns.tolist()[j] for j in np.argwhere(labels == i).T[0]] for i in np.unique(labels)]
    return grouped_columns

def feature_explore_examples():
    df_path = 'C:\\Users\\Clint\\PycharmProjects\\BigDataEnergy\\ml\\data\\ShapeUp\\pdDataStorage_v4.xlsx'
    df_original = pd.read_excel(df_path)

    df_ext = require_extracted(df_path, all_features+all_targets)
    corr_df = plot_correlation(df_ext, all_features+['DEM_SEX'], all_targets)

    X = corr_df.values
    d = sch.distance.pdist(X)  # vector of (n choose 2) pairwise distances
    L = sch.linkage(d, method='complete')
    ind = sch.fcluster(L, 0.5 * d.max(), 'distance')
    del df_ext['SubjectID']
    del df_ext['SubjectID.1']
    del df_ext['DEM_SEX']
    # plot_correlation(df_ext, list(corr_cluster(df_ext)), all_targets)
    # corr_df = plot_correlation(df_ext, columns, all_targets)

    dissimilarity = 1 - np.abs(corr_df)
    hierarchy = sch.linkage(squareform(1 - np.abs(corr_df)), method='average')
    labels = sch.fcluster(hierarchy, 0.5, criterion='distance')
    print()

    # feature_explore(df_original, 'DA_3DO3_CIRC_Ch')
    # df_data = feature_explore(df_original, m_common+a_b_all, age_groups=True)
    # print(df_data)
    #
    # print(RefValueTable(df_original[m_common+m_manual]))
    # print(df_original[m_common+m_manual].describe())  # default describe funtion from pandas

if __name__ == "__main__":
    feature_explore_examples()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Sep 13 08:59:42 2019

@author: Rajiv Sambasivan
"""
from math import ceil

import numpy as np
import pandas as pd
import torch
from sklearn.preprocessing import StandardScaler
from torch.utils import data

class CH_Dataset(data.Dataset):
    "Characterizes a dataset for PyTorch"

    def __init__(self, fp="cal_housing.csv", train=True, trng_prop=0.667):
        "Initialization"
        self.file_path = fp
        df = pd.read_csv(self.file_path)
        preds = df.columns.tolist()
        preds.remove("medianHouseValue")
        df["medianHouseValue"] = np.log(df["medianHouseValue"])
        featureset = df.dtypes.to_dict()
        featureset = {k: str(featureset[k]) for k in featureset}
        featureset["name"] = "log_transformed_median_house_value"
        self.featureset = featureset
        self.ds_info = {
            "name": "california-housing-dataset",
            "description": "This dataset lists median house prices in Califoria. Various house features are provided",
            "source": "UCI ML Repository",
        }
        df[preds] = StandardScaler().fit_transform(df[preds].values)
        num_rows = df.shape[0] - 1
        trng_end = ceil(num_rows * trng_prop)
        if train:
            self.X = torch.from_numpy(df.loc[:trng_end, preds].values)
            self.Y = torch.from_numpy(df.loc[:trng_end, "medianHouseValue"].values)
        else:
            self.X = torch.from_numpy(df.loc[(trng_end + 1) :, preds].values)
            self.Y = torch.from_numpy(
                df.loc[(trng_end + 1) :, "medianHouseValue"].values
            )
        self.input_size = len(preds)
        self.output_size = 1

    def __len__(self):
        "Denotes the total number of samples"
        return len(self.X)

    def __getitem__(self, index):
        "Generates one sample of data"
        # Load data and get label
        return self.X[index], self.Y[index]

    def get_featureset(self):
        return self.featureset

    def get_dataset(self):
        return self.ds_info


import pygame
from sys import exit

pygame.init()
screen = pygame.display.set_mode((800, 400))
pygame.display.set_caption("Platformer Game")
clock = pygame.time.Clock()
bg = pygame.Surface((800, 400))
ground = pygame.Surface((800, 200))
font = pygame.font.Font('font/nova.ttf', 30)
enemy = pygame.Surface((50, 50))
player = pygame.Surface((50, 100))

enemyOriginalX = 600
enemyOriginalY = 250
enemySpeed = 4
enemy_rect = enemy.get_rect(topleft = (enemyOriginalX, enemyOriginalY))
player_rect = player.get_rect(topleft = (20, 200))

bg.fill('blue')
ground.fill('green')
title = font.render('Platformer Game', False, 'yellow')
enemy.fill('purple')

while True:
  for event in pygame.event.get():
    if event.type == pygame.QUIT:
      pygame.quit()
      exit()
  screen.blit(bg, (0, 0))
  screen.blit(ground, (0, 300))
  screen.blit(title, (275, 50))
  screen.blit(enemy, enemy_rect)
  screen.blit(player, player_rect)
  enemy_rect.x -= enemySpeed
  if enemy_rect.right <= 0: enemy_rect.left = 800

  #if player_rect.colliderect(enemy_rect):
    #print('epic')
  mouse_pos = pygame.mouse.get_pos()
  if player_rect.collidepoint((mouse_pos)):
    print(pygame.mouse.get_pressed())

  pygame.display.update()
  clock.tick(60)


from collections import deque

nums = list(map(int, input().split(",")))
nums_list = list(enumerate(nums))

# 队列的使用
queue = deque()
for num in nums_list:
    queue.append((num[0], num[1]))

ans, count = list(), 0
while True:
    if len(queue) <= 1:
        break
    task = queue.popleft()
    cur_max = max(queue, key=lambda x: x[1])
    if task[1] >= cur_max[1]:
        ans.append((task[0], count))
        count += 1
    else:
        queue.append(task)
ans.append((queue[0][0], count))

print(",".join(map(lambda x: str(x[1]), sorted(ans, key=lambda x: x[0]))))

11/Python 实现【打印任务排序】/Python 实现【打印任务排序】.py

Python 实现【打印任务排序】.py


import numpy as np
import os
import sys
import re
import json
from scipy.spatial.transform import Rotation as R
import shutil
import struct

#################################
#   General Helper Functions    #
#################################

def rotate_positions_around_z(positions):
    """
    Given a list of coordinates, rotate them all by 90 degrees around the z-axis. This is used to
        convert particle coordinates from the raw tiltseries to the final reconstruction's
        coordinate system for simulated data.

    Args:
        positions: A list of [x, y, z] coordinates

    Returns: None

    """
    rot = R.from_euler("zxz", (90, 0, 0), degrees=True)
    for i, point in enumerate(positions):
        positions[i] = np.dot(rot.as_matrix(), np.array(point))

    return positions

def convert_slicer_to_motl(orientations):
    """
    Convert a set of Slicer angles to the reference-to-particle ZXZ, external Euler angles for Artiatomi
    Args:
        orientations: The list of Slicer angles

    Returns: The list of Euler angles

    """
    for i, point in enumerate(orientations):
        slicer = orientations[i]
        # Rotate by 90 around the x-axis so that the membrane is in the XY
        # plane (Phi gives in-plane rotation)
        slicer_rot = R.from_euler(
            "zyx", [slicer[2], slicer[1], slicer[0]], degrees=True
        )
        orientation_mat = np.dot(
            R.from_euler("zxz", [0, 90, 0], degrees=True).as_matrix(),
            slicer_rot.as_matrix(),
        )
        rotation = R.from_matrix(orientation_mat)
        ref_to_part = rotation.inv()
        eulers = ref_to_part.as_euler("zxz", degrees=True)
        # Like PEET, Artiatomi takes Z1, Z2, X
        orientations[i] = [eulers[0], eulers[2], eulers[1]]

    return orientations

def get_slicer_info(mod_file):
    """
    Open an IMOD .mod file and retrieve the Slicer information

    Args:
        mod_file: The .mod file path

    Returns: A list of Slicer point objects with keys {"angles", "coords"}

    """
    results = []
    with open(mod_file, "rb") as file:
        token = file.read(4)
        if token != b"IMOD":
            print(
                "ID of .mod file is not 'IMOD'. This does not seem to be an IMOD MOD file!"
            )
            exit(1)

        # Read past rest of ID and file header
        file.read(236)

        while token != b"IEOF":
            file.seek(-3, 1)
            token = file.read(4)
            if token == b"SLAN":
                # Read past SLAN object size and time
                file.read(8)

                angles = struct.unpack(">" + ("f" * 3), file.read(4 * 3))
                xyz = struct.unpack(">" + ("f" * 3), file.read(4 * 3))

                results.append({"angles": angles, "coords": xyz})
                file.read(32)

                # Read forward a little so next iteration of while loop starts at end of SLAN object
                file.read(3)
            elif token == b"OBJT":
                # Objects are 176 bytes; skip path to make reading faster
                file.read(176)
                # Read forward a little so next iteration of while loop starts at end of object
                file.read(3)

    if len(results) == 0:
        print("Reached end of MOD file without finding slicer angles!")
        exit(1)

    return results

#######################
#   IMOD Functions    #
#######################

def imod_setup_sta(artia_root, dirs_start_with):
    """
    Write out the info file necessary to run the MATLAB set up script for sub-tomogram averaging in Artiatomi
    Args:
        artia_root: The Artiatomi project directory
        dirs_start_with: A prefix for the data subdirectories within the Artiatomi project

    Returns: Text filepath containing the parsed info

    """

    lines = ["Directory Tomonum Motl\n"]
    for subdir in os.listdir(artia_root):
        if subdir.startswith(dirs_start_with):
            motl_name = ""
            full_subdir = os.path.join(artia_root, subdir)
            for file in os.listdir(full_subdir):
                if file.endswith(".st"):
                    basename = file.split(".")[0]
                    motl_name = f"{basename}_motl.em"
                    break

            motl = os.path.join(artia_root, subdir, motl_name)
            tomo_num = int(subdir.split("_")[-1]) + 1

            lines.append(f"{full_subdir} {tomo_num} {motl}\n")

    info_file = os.path.join(artia_root, "tomo_motls.txt")

    with open(info_file, "w") as f:
        f.writelines(lines)

    return info_file

def shift_coordinates_bottom_left(coords, size, binning=1):
    """
    Given an XYZ tuple of particle coordinates and the reconstruction they came from, shift the
        coordinates so that the origin is at the bottom-left of the tomogram

    Args:
        coords: the (x, y, z) coordinates for the particle
        size: the reconstruction MRC half-dimensions in (nx/2, ny/2, nz/2) form
        binning: the bin factor from the original stack to the final reconstruction, to be used if
            you are using coordinates based on the original unbinned coordinate system

    Returns: the new coordinates as a (x, y, z) tuple

    """
    return (
        float(coords[0]) / binning + size[0],
        float(coords[1]) / binning + size[1],
        float(coords[2]) / binning + size[2],
    )

def write_out_motl_files_simulated(root, name, xyz_name, motl_name, size, binning):
    """
    Iterate through an IMOD Processor project and set up text files for Artiatomi to read in for its marker files

    Args:
        root: The ETSimulations project root
        name: The project/particle name
        xyz_name: The file name to give each XYZ coordinates file
        motl_name: The file name to give each Euler angles file
        size: Final tomogram size (for shifting origin of coordinates)
        binning: The binning to apply to the raw coordinates

    Returns: None
    """
    metadata_file = os.path.join(root, "sim_metadata.json")

    with open(metadata_file, "r") as f:
        metadata = json.loads(f.read())

        # -------------------------------------
        # Retrieve parameters to write to files
        # -------------------------------------
        for num, tomogram in enumerate(metadata):
            basename = "%s_%d" % (name, tomogram["global_stack_no"])
            tomogram_dir = os.path.join(root, "processed_data/IMOD", basename)

            artia_dir = os.path.join(root, "processed_data/Artiatomi", basename)

            # Positions for TEM-Simulator are in nm, need to convert to pixels
            positions = np.array(tomogram["positions"]) / tomogram["apix"]
            # During reconstruction, there is a 90 degree rotation around the z-axis, so correct for
            # that with the positions
            positions = rotate_positions_around_z(positions)

            slicer_angles_csv = os.path.join(tomogram_dir, "%s_slicerAngles.csv" % name)
            orientations = np.loadtxt(slicer_angles_csv, delimiter=",")
            eulers = convert_slicer_to_motl(orientations)
            # Compile Slicer infos
            slicer_info = []
            for i, coords in enumerate(positions):
                angles = eulers[i]
                slicer_info.append({"coords": coords, "angles": angles})

            # Write out MOTL files for each tomogram
            xyz_motl = os.path.join(artia_dir, xyz_name)
            eulers_motl = os.path.join(artia_dir, motl_name)
            xyz_motl_file = open(xyz_motl, "w")
            eulers_motl_file = open(eulers_motl, "w")
            for info in slicer_info:
                coords = shift_coordinates_bottom_left(info["coords"], size, binning)
                angles = info["angles"]
                xyz_line = "{:f} {:f} {:f}\n".format(coords[0], coords[1], coords[2])
                xyz_motl_file.write(xyz_line)
                motl_line = "{:f} {:f} {:f}\n".format(angles[0], angles[1], angles[2])
                eulers_motl_file.write(motl_line)

            xyz_motl_file.close()
            eulers_motl_file.close()

def write_out_motl_files_real(artia_root, xyz_name, motl_name):
    """
    Iterate through Artiatomi files copied over from IMOD real data and set up text files for Artiatomi to read in for
        its marker files based on slicer .mod files

    Args:
        artia_root: The Artiatomi project root
        xyz_name: The file name to give each XYZ coordinates file
        motl_name: The file name to give each Euler angles file

    Returns: None
    """
    for subdir in os.listdir(artia_root):
        # Find the mod file
        mod_file = ""
        for file in os.listdir(os.path.join(artia_root, subdir)):
            if file.endswith(".mod"):
                mod_file = os.path.join(artia_root, subdir, file)
                break

        if mod_file == "":
            print("ERROR: No mod was found for sub-directory: %s" % subdir)
            exit(1)

        slicer_info = get_slicer_info(mod_file)

        # Write out MOTL files for each tomogram
        xyz_motl = os.path.join(artia_root, subdir, xyz_name)
        eulers_motl = os.path.join(artia_root, subdir, motl_name)
        xyz_motl_file = open(xyz_motl, "w")
        eulers_motl_file = open(eulers_motl, "w")
        for info in slicer_info:
            coords = info["coords"]
            angles = info["angles"]
            xyz_line = "{:f} {:f} {:f}\n".format(coords[0], coords[1], coords[2])
            xyz_motl_file.write(xyz_line)
            motl_line = "{:f} {:f} {:f}\n".format(angles[0], angles[2], angles[1])
            eulers_motl_file.write(motl_line)

        xyz_motl_file.close()
        eulers_motl_file.close()

def copy_over_imod_files(
    imod_root, artia_root, dir_starts_with, real_data_mode=False, mod_contains=None
):
    for subdir in os.listdir(imod_root):
        if subdir.startswith(dir_starts_with):

            # Look for the necessary IMOD files
            xf = ""
            tlt = ""
            stack = ""
            mod = ""
            if not real_data_mode:
                mod = "none"
            for file in os.listdir(os.path.join(imod_root, subdir)):
                if file.endswith(".tlt") and not file.endswith("_fid.tlt"):
                    tlt = file
                elif file.endswith(".st"):
                    stack = file
                elif file.endswith(".xf") and not file.endswith("_fid.xf"):
                    xf = file
                if real_data_mode and mod_contains in file and file.endswith(".mod"):
                    mod = file

                # Break out of loop once all three relevant files have been found
                if tlt != "" and stack != "" and xf != "" and mod != "":
                    break

            artia_stack_dir = os.path.join(artia_root, subdir)
            if not os.path.exists(artia_stack_dir):
                os.mkdir(artia_stack_dir)

            # Copy over the stack to the Artiatomi folder
            if stack != "":
                shutil.copyfile(
                    os.path.join(imod_root, subdir, stack),
                    os.path.join(artia_stack_dir, stack),
                )
            else:
                print("ERROR: No tiltseries was found for sub-directory: %s" % subdir)
                exit(1)

            # Copy over the tlt file to the Artiatomi folder
            if tlt != "":
                shutil.copyfile(
                    os.path.join(imod_root, subdir, tlt),
                    os.path.join(artia_stack_dir, tlt),
                )
            else:
                print("WARNING: No .tlt file was found for sub-directory: %s" % subdir)

            # Copy over the tlt file to the Artiatomi folder
            if xf != "":
                shutil.copyfile(
                    os.path.join(imod_root, subdir, xf),
                    os.path.join(artia_stack_dir, xf),
                )
            else:
                print("WARNING: No .xf file was found for sub-directory: %s" % subdir)

            if real_data_mode:
                # Copy over the mod file to the Artiatomi folder
                if mod != "":
                    shutil.copyfile(
                        os.path.join(imod_root, subdir, mod),
                        os.path.join(artia_stack_dir, mod),
                    )
                else:
                    print(
                        "WARNING: No .mod file was found for sub-directory: %s" % subdir
                    )

def setup_reconstructions_script(root, name, artia_args):
    """
    Function to generate the Artiatomi reconstruction setup script

    Args:
        root: The ETSimulations project root (only used for coming from simulated data)
        name: The project/particle name (only used for coming from simulated data)
        artia_args: The Artiatomi Processor arguments

    Returns: None

    """
    xyz_motl = "%s_xyz.txt" % name
    eulers_motl = "%s_motl.txt" % name

    # Get input parameters based on mode
    if artia_args["real_data_mode"]:
        artia_root = artia_args["artia_dir"]
        imod_root = artia_args["imod_dir"]
        dir_starts_with = artia_args["dir_starts_with"]
        mod_contains = artia_args["mod_contains"]
    else:
        artia_root = os.path.join(root, "processed_data", "Artiatomi")
        imod_root = os.path.join(root, "processed_data", "IMOD")
        dir_starts_with = name
        mod_contains = ""

    # -------------------------------------
    # Set up Artiatomi project directory structure
    # -------------------------------------
    print("Creating Artiatomi project directories and copying over relevant IMOD files")
    if not os.path.exists(artia_root):
        os.mkdir(artia_root)

    copy_over_imod_files(
        imod_root,
        artia_root,
        dir_starts_with,
        real_data_mode=artia_args["real_data_mode"],
        mod_contains=mod_contains,
    )

    print("Writing out MOTL-related files")
    if artia_args["real_data_mode"]:
        write_out_motl_files_real(artia_root, xyz_motl, eulers_motl)
    else:
        binning = 1
        if "position_binning" in artia_args:
            binning = artia_args["position_binning"]

        size = (
            artia_args["tomogram_size_x"] / 2,
            artia_args["tomogram_size_y"] / 2,
            artia_args["tomogram_size_z"] / 2,
        )
        write_out_motl_files_simulated(root, name, xyz_motl, eulers_motl, size, binning)

    # Use template file to create Matlab script to run the remaining steps
    current_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
    template = current_dir + "../../templates/artiatomi/setup_artia_reconstructions.m"
    template_path = os.path.realpath(template)
    new_script = os.path.join(artia_root, "setup_artia_reconstructions.m")
    print("")
    print("Creating processing script at: %s" % new_script)

    with open(new_script, "w") as new_file:
        with open(template_path, "r") as base_file:
            # First look for the input params section
            while True:
                line = base_file.readline()
                if re.match(r"^%% Input parameters", line):
                    break
                else:
                    new_file.write(line)

            # Now start replacing input params
            while True:
                line = base_file.readline()
                # Break once we reach the end of the segment
                if re.match(r"^%% Process", line):
                    break

                # If we are at an assignment line
                elif re.match(r".+ =", line):
                    line = line.strip()
                    tokens = line.split(" ")
                    variable_name = tokens[0]

                    value_to_write_out = ""
                    if variable_name == "project_root":
                        value_to_write_out = f"'{artia_root}';"
                    elif variable_name == "dir_starts_with":
                        value_to_write_out = f"'{dir_starts_with}';"
                    elif variable_name == "xyz_motl":
                        value_to_write_out = f"'{xyz_motl}';"
                    elif variable_name == "eulers_motl":
                        value_to_write_out = f"'{eulers_motl}';"
                    elif variable_name in artia_args:
                        if type(artia_args[variable_name]) == str:
                            value_to_write_out = f"'{artia_args[variable_name]}';"
                        else:
                            value_to_write_out = str(artia_args[variable_name]) + ";"
                    else:
                        print(
                            "Missing Artiatomi processing parameter: %s!"
                            % variable_name
                        )
                        exit(1)

                    new_line = " ".join([variable_name, "=", value_to_write_out, "\n"])

                    new_file.write(new_line)

                # Other lines in the segment - probably just comments
                else:
                    new_file.write(line)

            # For the rest of the code, just write it out
            while True:
                line = base_file.readline()
                if len(line) == 0:
                    break
                else:
                    new_file.write(line)

def generate_reconstructions_script(root, name, artia_args):
    """
    Generate a bash script to run the reconstructions.
    Args:
        root: ETSimulations root (only used if coming from simulated data)
        name: Project/particle name (only used if coming from simulated data)
        artia_args: The Artiatomi processor arguments

    Returns: None

    """
    if artia_args["real_data_mode"]:
        artia_root = artia_args["artia_dir"]
        dir_starts_with = artia_args["dir_starts_with"]
    else:
        artia_root = os.path.join(root, "processed_data", "Artiatomi")
        dir_starts_with = name

    # Use template file to create a bash script to run EmSART on a data set
    current_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
    template = current_dir + "../../templates/artiatomi/emsart_reconstruct.sh"
    template_path = os.path.realpath(template)
    new_script = os.path.join(artia_root, "emsart_reconstruct.sh")
    print("")
    print("Creating processing script at: %s" % new_script)

    with open(new_script, "w") as new_file:
        with open(template_path, "r") as base_file:
            while True:
                line = base_file.readline()
                if len(line) == 0:
                    break
                elif line.startswith("for f in"):
                    dir_pattern = f"{artia_root}/{dir_starts_with}*"
                    new_line = f"for f in {dir_pattern}\n"
                    new_file.write(new_line)
                elif line.startswith("config_file"):
                    config_file = os.path.basename(
                        artia_args["reconstruction_template_config"]
                    )
                    new_line = f'config_file="{config_file}"\n'
                    new_file.write(new_line)
                elif line.startswith("emsart_path"):
                    emsart = artia_args["emsart_path"]
                    new_line = f'emsart_path="{emsart}"\n'
                    new_file.write(new_line)
                else:
                    new_file.write(line)

def generate_sta_script(artia_root, info_file, artia_args):
    """
    Generate the MATLAB script for setting up the Artiatomi sub-tomogram averaging
    Args:
        artia_root: Artiatomi project root directory
        info_file: The text file listing the tomograms, tomogram numbers, and their motivelists
        artia_args: The Artiatomi processor arguments

    Returns: None

    """
    sta_folder = os.path.join(artia_root, "sta")
    mask_file = os.path.join(sta_folder, "other", "mask.em")
    wedge_file = os.path.join(sta_folder, "other", "wedge.em")
    maskCC_file = os.path.join(sta_folder, "other", "maskCC.em")
    global_motl_file = os.path.join(sta_folder, "motls", "motl_1.em")
    motl_file_pre = os.path.join(sta_folder, "motls", "motl_")
    particles_folder = os.path.join(sta_folder, "parts")
    part_file_pre = os.path.join(sta_folder, "parts", "part_")
    cfg_file = os.path.join(sta_folder, "sta.cfg")

    # Use template file to create Matlab script to run the remaining steps
    current_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
    template = current_dir + "../../templates/artiatomi/setup_artia_sta.m"
    template_path = os.path.realpath(template)
    new_script = os.path.join(artia_root, "setup_artia_sta.m")
    print("")
    print("Creating STA processing script at: %s" % new_script)

    with open(new_script, "w") as new_file:
        with open(template_path, "r") as base_file:
            # First look for the input params section
            while True:
                line = base_file.readline()
                if re.match(r"^%% Input parameters", line):
                    break
                else:
                    new_file.write(line)

            # Now start replacing input params
            while True:
                line = base_file.readline()
                # Break once we reach the end of the segment
                if re.match(r"^%% Load motl", line):
                    break

                # If we are at an assignment line
                elif re.match(r".+ =", line):
                    line = line.strip()
                    tokens = line.split(" ")
                    variable_name = tokens[0]

                    value_to_write_out = ""
                    if variable_name == "info_file":
                        value_to_write_out = f"'{info_file}';"
                    elif variable_name == "sta_folder":
                        value_to_write_out = f"'{sta_folder}';"
                    elif variable_name == "maskFile":
                        value_to_write_out = f"'{mask_file}';"
                    elif variable_name == "wedgeFile":
                        value_to_write_out = f"'{wedge_file}';"
                    elif variable_name == "maskCCFile":
                        value_to_write_out = f"'{maskCC_file}';"
                    elif variable_name == "motlFile":
                        value_to_write_out = f"'{global_motl_file}';"
                    elif variable_name == "motlFilePre":
                        value_to_write_out = f"'{motl_file_pre}';"
                    elif variable_name == "particles_folder":
                        value_to_write_out = f"'{particles_folder}';"
                    elif variable_name == "partFilePre":
                        value_to_write_out = f"'{part_file_pre}';"
                    elif variable_name == "avgCfgFile":
                        value_to_write_out = f"'{cfg_file}';"
                    elif variable_name in artia_args:
                        if type(artia_args[variable_name]) == str:
                            value_to_write_out = f"'{artia_args[variable_name]}';"
                        else:
                            value_to_write_out = str(artia_args[variable_name]) + ";"
                    else:
                        print(
                            "Missing Artiatomi processing parameter: %s!"
                            % variable_name
                        )
                        exit(1)

                    new_line = " ".join([variable_name, "=", value_to_write_out, "\n"])

                    new_file.write(new_line)

                # Other lines in the segment - probably just comments
                else:
                    new_file.write(line)

            # For the rest of the code, just write it out
            while True:
                line = base_file.readline()
                if len(line) == 0:
                    break
                else:
                    new_file.write(line)

def get_latest_ref(sta_dir):
    """
    Find the latest reference from an Artiatomi STA run
    Args:
        sta_dir: The sub-tomogram averaging directory, where we expect to find a ref folder

    Returns: The latest reference file path

    """
    ref_dir = os.path.join(sta_dir, "ref")
    max_num = 0
    latest_ref = None
    for ref in os.listdir(ref_dir):
        if re.match("ref[0-9]+.em", ref):
            num = int(ref.split("ref")[1].split(".")[0])
            if num > max_num:
                latest_ref = os.path.join(ref_dir, ref)

    return latest_ref

def get_latest_motl(sta_dir):
    """
    Find the latest motl from an Artiatomi STA run
    Args:
        sta_dir: The sub-tomogram averaging directory, where we expect to find a ref folder

    Returns: The latest motl file path

    """
    motls_dir = os.path.join(sta_dir, "motls")
    max_num = 0
    latest_motl = None
    for motl in os.listdir(motls_dir):
        if re.match("motl_[0-9]+.em", motl):
            num = int(motl.split("_")[1].split(".")[0])
            if num > max_num:
                latest_motl = os.path.join(motls_dir, motl)

    return latest_motl

def generate_refinement_script(artia_root, info_file, artia_args):
    """
    Generate the MATLAB script for setting up the Artiatomi tilt refinement
    Args:
        artia_root: Artiatomi project root directory
        info_file: The text file listing the tomograms, tomogram numbers, and their motivelists
        artia_args: The Artiatomi processor arguments

    Returns: None

    """

    mask_file = os.path.join(artia_root, "sta", "other", "mask.em")
    wedge_file = os.path.join(artia_root, "sta", "other", "wedge.em")
    maskCC_file = os.path.join(artia_root, "sta", "other", "maskCC.em")

    refine_dir = os.path.join(artia_root, "refine")
    refine_motls_dir = os.path.join(refine_dir, "motls")

    latest_ref = get_latest_ref(os.path.join(artia_root, "sta"))
    latest_motl = get_latest_motl(os.path.join(artia_root, "sta"))

    # Use template file to create Matlab script to run the remaining steps
    current_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
    template = current_dir + "../../templates/artiatomi/refine_align.m"
    template_path = os.path.realpath(template)
    new_script = os.path.join(artia_root, "refine_align.m")
    print("")
    print("Creating refinement script at: %s" % new_script)

    with open(new_script, "w") as new_file:
        with open(template_path, "r") as base_file:
            # First look for the input params section
            while True:
                line = base_file.readline()
                if re.match(r"^%% Input parameters", line):
                    break
                else:
                    new_file.write(line)

            # Now start replacing input params
            while True:
                line = base_file.readline()
                # Break once we reach the end of the segment
                if re.match(r"^%% Split latest", line):
                    break

                # If we are at an assignment line
                elif re.match(r".+ =", line):
                    line = line.strip()
                    tokens = line.split(" ")
                    variable_name = tokens[0]

                    value_to_write_out = ""
                    if variable_name == "info_file":
                        value_to_write_out = f"'{info_file}';"
                    elif variable_name == "mask_file":
                        value_to_write_out = f"'{mask_file}';"
                    elif variable_name == "wedge_file":
                        value_to_write_out = f"'{wedge_file}';"
                    elif variable_name == "maskCC_file":
                        value_to_write_out = f"'{maskCC_file}';"
                    elif variable_name == "main_root":
                        value_to_write_out = f"'{refine_dir}';"
                    elif variable_name == "refine_motls_dir":
                        value_to_write_out = f"'{refine_motls_dir}';"
                    elif variable_name == "latest_ref":
                        value_to_write_out = f"'{latest_ref}';"
                    elif variable_name == "latest_motl":
                        value_to_write_out = f"'{latest_motl}';"
                    elif variable_name in artia_args:
                        if type(artia_args[variable_name]) == str:
                            value_to_write_out = f"'{artia_args[variable_name]}';"
                        else:
                            value_to_write_out = str(artia_args[variable_name]) + ";"
                    else:
                        print(
                            "Missing Artiatomi processing parameter: %s!"
                            % variable_name
                        )
                        exit(1)

                    new_line = " ".join([variable_name, "=", value_to_write_out, "\n"])

                    new_file.write(new_line)

                # Other lines in the segment - probably just comments
                else:
                    new_file.write(line)

            # For the rest of the code, just write it out
            while True:
                line = base_file.readline()
                if len(line) == 0:
                    break
                else:
                    new_file.write(line)

def generate_extract_script(artia_root, artia_args):
    """
    Generate the MATLAB script for extracting new particles based on local refinement
    Args:
        artia_root: Artiatomi project root directory
        artia_args: The Artiatomi processor arguments

    Returns: None

    """
    refine_dir = os.path.join(artia_root, "refine")
    subvol_pre = refine_dir + "/subvols/"
    latest_motl = get_latest_motl(os.path.join(artia_root, "sta"))

    mask_file = os.path.join(artia_root, "sta", "other", "mask.em")
    wedge_file = os.path.join(artia_root, "sta", "other", "wedge.em")
    maskCC_file = os.path.join(artia_root, "sta", "other", "maskCC.em")

    # Use template file to create Matlab script to run the remaining steps
    current_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
    template = current_dir + "../../templates/artiatomi/refine_extract.m"
    template_path = os.path.realpath(template)
    new_script = os.path.join(artia_root, "refine_extract.m")
    print("")
    print("Creating refinement extraction script at: %s" % new_script)

    with open(new_script, "w") as new_file:
        with open(template_path, "r") as base_file:
            # First look for the input params section
            while True:
                line = base_file.readline()
                if re.match(r"^%% Input parameters", line):
                    break
                else:
                    new_file.write(line)

            # Now start replacing input params
            while True:
                line = base_file.readline()
                # Break once we reach the end of the segment
                if re.match(r"^%% Run extractions", line):
                    break

                # If we are at an assignment line
                elif re.match(r".+ =", line):
                    line = line.strip()
                    tokens = line.split(" ")
                    variable_name = tokens[0]

                    value_to_write_out = ""
                    if variable_name == "refineDir":
                        value_to_write_out = f"'{refine_dir}';"
                    elif variable_name == "subVolPre":
                        value_to_write_out = f"'{subvol_pre}';"
                    elif variable_name == "latestStaMotl":
                        value_to_write_out = f"'{latest_motl}';"
                    elif variable_name == "mask_file":
                        value_to_write_out = f"'{mask_file}';"
                    elif variable_name == "wedge_file":
                        value_to_write_out = f"'{wedge_file}';"
                    elif variable_name == "maskCC_file":
                        value_to_write_out = f"'{maskCC_file}';"
                    elif variable_name in artia_args:
                        if type(artia_args[variable_name]) == str:
                            value_to_write_out = f"'{artia_args[variable_name]}';"
                        else:
                            value_to_write_out = str(artia_args[variable_name]) + ";"
                    else:
                        print(
                            "Missing Artiatomi processing parameter: %s!"
                            % variable_name
                        )
                        exit(1)

                    new_line = " ".join([variable_name, "=", value_to_write_out, "\n"])

                    new_file.write(new_line)

                # Other lines in the segment - probably just comments
                else:
                    new_file.write(line)

            # For the rest of the code, just write it out
            while True:
                line = base_file.readline()
                if len(line) == 0:
                    break
                else:
                    new_file.write(line)

def artiatomi_main(root, name, artia_args):

    if (
        "setup_reconstructions_and_motls" in artia_args
        and artia_args["setup_reconstructions_and_motls"]
    ):
        setup_reconstructions_script(root, name, artia_args)

        generate_reconstructions_script(root, name, artia_args)

    if "setup_averaging" in artia_args and artia_args["setup_averaging"]:

        # Get input parameters based on mode
        if artia_args["real_data_mode"]:
            artia_root = artia_args["artia_dir"]
            dir_starts_with = artia_args["dir_starts_with"]
        else:
            artia_root = os.path.join(root, "processed_data", "Artiatomi")
            dir_starts_with = name

        # Set up averaging directory structure
        print("Creating Artiatomi averaging directories")
        sta_dir = os.path.join(artia_root, "sta")
        if not os.path.exists(sta_dir):
            os.mkdir(sta_dir)

        parts_dir = os.path.join(sta_dir, "parts")
        if not os.path.exists(parts_dir):
            os.mkdir(parts_dir)

        motls_dir = os.path.join(sta_dir, "motls")
        if not os.path.exists(motls_dir):
            os.mkdir(motls_dir)

        ref_dir = os.path.join(sta_dir, "ref")
        if not os.path.exists(ref_dir):
            os.mkdir(ref_dir)

        others_dir = os.path.join(sta_dir, "other")
        if not os.path.exists(others_dir):
            os.mkdir(others_dir)

        info_file = imod_setup_sta(artia_root, dir_starts_with)

        generate_sta_script(artia_root, info_file, artia_args)

    if "setup_refinement" in artia_args and artia_args["setup_refinement"]:

        # Get input parameters based on mode
        if artia_args["real_data_mode"]:
            artia_root = artia_args["artia_dir"]
        else:
            artia_root = os.path.join(root, "processed_data", "Artiatomi")

        info_file = os.path.join(artia_root, "tomo_motls.txt")

        # Set up averaging directory structure
        print("Creating Artiatomi refinement directories")
        refine_dir = os.path.join(artia_root, "refine")
        if not os.path.exists(refine_dir):
            os.mkdir(refine_dir)

        motls_dir = os.path.join(refine_dir, "motls")
        if not os.path.exists(motls_dir):
            os.mkdir(motls_dir)

        generate_refinement_script(artia_root, info_file, artia_args)

        generate_extract_script(artia_root, artia_args)


import click
import os
import torch
import torch.cuda
from torch.nn import functional as F
from torch.optim import Adam
from torch.utils.data import DataLoader

from util.dataset import DADataset, load_source_target_datasets
from util.io import get_config, save_model
from util.net import LenetClassifier, weights_init
from util.opt import exp_list
from util.preprocess import get_composed_transforms

torch.backends.cudnn.benchmark = True

@click.command()
@click.option('--exp', type=click.Choice(exp_list), required=True)
@click.option('--use_affine', is_flag=True)
@click.option('--num_epochs', type=int, default=200)
@click.option('--train_type', type=click.Choice(['sup', 'unsup']))
def experiment(exp, use_affine, num_epochs, train_type):
    config = get_config('config.yaml')
    lr = float(config['lr'])
    weight_decay = float(config['weight_decay'])
    batch_size = int(config['batch_size'])

    log_dir = 'log/{:s}/{:s}'.format(exp, train_type)
    os.makedirs(log_dir, exist_ok=True)
    device = torch.device('cuda')

    src, tgt = load_source_target_datasets(exp)
    train = tgt if train_type == 'sup' else src
    test = tgt

    n_ch = train.train_X.shape[1]  # number of color channels
    res = train.train_X.shape[-1]  # size of image
    n_classes = train.n_classes

    cls = LenetClassifier(n_classes, n_ch, res).to(device)
    cls.apply(weights_init('kaiming'))

    optimizer = Adam(list(cls.parameters()), lr=lr, weight_decay=weight_decay)

    train_tfs = get_composed_transforms(train=True)
    test_tfs = get_composed_transforms(train=False)
    train_data = DADataset(train.train_X, train.train_y, train_tfs, use_affine)
    test_data = DADataset(test.test_X, test.test_y, test_tfs, use_affine)

    train_loader = DataLoader(train_data, batch_size=batch_size,
                              num_workers=4, shuffle=True)
    test_loader = DataLoader(test_data, batch_size=batch_size, num_workers=4)

    print('Training...')
    for epoch in range(1, num_epochs + 1):

        cls.train()
        for tgt_x, tgt_y in train_loader:
            loss = F.cross_entropy(cls(tgt_x.to(device)), tgt_y.to(device))
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        if epoch % 5 == 0 and epoch > 0:
            cls.eval()
            n_err = 0
            with torch.no_grad():
                for tgt_x, tgt_y in test_loader:
                    prob_y = F.softmax(cls(tgt_x.to(device)), dim=1)
                    pred_y = torch.max(prob_y, dim=1)[1]
                    pred_y = pred_y.to(torch.device('cpu'))
                    n_err += (pred_y != tgt_y).sum().item()
            print('Epoch {:d}, Err {:f}'.format(epoch, n_err / len(test_data)))

        if epoch % 100 == 0 and epoch > 0:
            save_model(cls, '{:s}/epoch{:d}.tar'.format(log_dir, epoch))

if __name__ == '__main__':
    experiment()


from django.shortcuts import render, redirect
import asyncio
import telethon
# from telethon import TelegramClient, events, sync
from telethon.tl.functions.channels import JoinChannelRequest
# from telegram.client import Telegram

import socks

# Create your views here.
api_id = 111111
api_hash = 'apifromtelegram'
# loop = asyncio.new_event_loop()
# asyncio.set_event_loop(loop)
# client = telethon.TelegramClient('anon', api_id, api_hash, loop=loop)

def main(request):
    #
    # async def send_telethon_message(client):
    #     me = await client.get_me()
    #     await client.send_message('me', 'Testing Telethon1')
    # client = TelegramClient('anon', api_id, api_hash, loop=loop)
    # with client:
    #     loop.run_until_complete(send_telethon_message(client))
    # k = 0

    # async def check():
    #     dialogs = await client.get_dialogs()
    #     print('dialogsinner')
    #     print(dialogs)
    #     return dialogs
    # dialogs = check()
    # print('dialogs')
    # print(dialogs)

    # dialogs = client.get_dialogs()
    # for chat in dialogs.chats:
    #     getmessage = client.get_messages(chat.id, limit=400)
    #     for message in getmessage:
    #         print(message.message)

    # client = TelegramClient('@BigZEcheese', api_id, api_hash)
    # client = TelegramClient('@BigZEcheese', api_id, api_hash,  loop=loop)
    # with TelegramClient('anon', api_id, api_hash,  loop=loop) as client:
    #     loop.run_until_complete(client.send_message('me', 'Hello, myself!'))
    # client.start()
    # phone_number = '+79080196349'
    # client.send_code_request(phone_number)
    # myself = client.sign_in(phone_number, input('Enter code: '))
    # async def main():
    #     # Now you can use all client methods listed below, like for example...
    #     await client.send_message('me', 'Hello to myself!')
    #
    # with client:
    #     client.loop.run_until_complete(main())
    # # print(client.get_me().stringify())
    #
    # client.send_message('me', 'Hello! Talking to you from Telethon')
    # client.send_message('@ayasmaa_artyna', 'Hello! Talking to you from Telethon')

    # tg = Telegram(
    #     api_id=api_id,
    #     api_hash=api_hash,
    #     phone='+89080196349',  # you can pass 'bot_token' instead
    #     database_encryption_key='admin',
    # )
    # tg.login()
    #
    # # if this is the first run, library needs to preload all chats
    # # otherwise the message will not be sent
    # result = tg.get_chats()
    # result.wait()
    #
    # print(result)
    # # `tdlib` is asynchronous, so `python-telegram` always returns you an `AsyncResult` object.
    # # You can receive a result with the `wait` method of this object.
    # result.wait()
    # print(result.update)
    #
    # tg.stop()  # you must call `stop` at the end of the script
    # client = TelegramClient('some_name',api_id,api_hash,proxy = (socks.SOCKS5, 'localhost', 8000)  # Настройки прокси
    # ).start()
    # # Some_name здесь - случайное имя, при первом запуске вы сможете ввести свой номер телефона и код подтверждения, а затем будет сгенерирован файл some_name.session, и вам не нужно вводить код подтверждения номера телефона. неоднократно, когда вы запускаете его снова
    # myself = client.get_me()
    # print(myself)  # Проверяем, подключаться ли

    def code_callback():
        while True:
            code = input()
            return code
    async def run():
        try:
            await client.connect()   #From DB check, существует ли файл, если да, то коннектимся, если нет, то создаём
        except:
            await client.start(phone='+79080196349',  code_callback=code_callback)
        # dialogs = await client.get_dialogs()
        # channel = await client.get_entity('me')
        # first = dialogs[0]
        # for i in range(0, 5):
        #     print(dialogs[i].title)
        # # i = 0
        # # async for message in client.iter_messages(channel):
        # #     i = i + 1
        # #     print(str(i))
        # #     print(message)
        # #     if i == 5:
        # #         break
        # result = await client(telethon.functions.messages.GetPeerDialogsRequest(
        #     peers=[first]
        # ))
        # async for message in client.iter_messages(first, limit = 5 ):
        #     print(message.id, message.text)
        # mes = len(await  client.get_messages(first, limit = 1))
        # print(mes)
        # print(result.dialogs[0].read_outbox_max_id)  # Мною отправленное письмо, прочитанное последнее
        # print(result.dialogs[0])
        # if result.dialogs[0].read_outbox_max_id >= mes:
        #     print('the message has been read')
        # else:
        #     print('the message has not been read yet')
        #
        # @client.on(telethon.events.NewMessage(chats=("CANNEL")))
        # async def handler(event):
        #     if 'user_id=2807' in str(event.message):
        #         if 'text' in str(event.message):
        #             await event.reply('MESSAGE')
        # await client.send_message('@gexos', 'Вот это сообщение отправил от Python')
        user_id_list = []
        # @client.on(telethon.events.NewMessage(incoming=True, forwards=None))
        # async def handler(event):
        #     user_info = event.message.to_dict()['from_id']
        #     user_id = user_info['user_id']
        #     count = 0
        #     for i in user_id_list:
        #         if i == user_id:
        #             count += 1
        #         else:
        #             continue
        #     if count == 0:
        #         user_id_list.append(user_id)
        #         await asyncio.sleep(5)
        #         await client.send_message(user_id, 'Техт')
        #     print(user_id_list)
        # response = client.wait_event(telethon.events.NewMessage(
        #     outgoing=True,
        #     forwards=None
        # ))
        # response = await response
        # code = response.message.message.strip()
        # print(code)
    global client
    global loop
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    client = telethon.TelegramClient('anon', api_id, api_hash, loop=loop)
    client.loop.run_until_complete(run())
    print(type(client))
    return redirect('wait')

# @client.on(telethon.events.NewMessage(incoming=True, forwards=None))
#     async def handler(event):
#         # user_info = event.message.to_dict()['from_id']
#         # user_id = user_info['user_id']
#         # count = 0
#         # for i in user_id_list:
#         #     if i == user_id:
#         #         count += 1
#         #     else:
#         #         continue
#         # if count == 0:
#         #     user_id_list.append(user_id)
#         #     await asyncio.sleep(5)
#         #     await client.send_message(user_id, 'Техт')
#         print('asdadsdassaxa')
def notmain(request):
    async def run():
        await  client.disconnect()
    print(type(client))
    client.loop.run_until_complete(run())
    # client.loop.run_until_complete(run())
    return render(request, 'index.html')

def wait(request):

    async def run():
        dialogs = await client.get_dialogs()
        channel = await client.get_entity('me')
        first = dialogs[0]
        for i in range(0, 5):
            print(dialogs[i].title)
        # i = 0
        # async for message in client.iter_messages(channel):
        #     i = i + 1
        #     print(str(i))
        #     print(message)
        #     if i == 5:
        #         break
        result = await client(telethon.functions.messages.GetPeerDialogsRequest(
            peers=[first]
        ))
        async for message in client.iter_messages(first, limit = 5 ):
            print(message.id, message.text)
        mes = len(await  client.get_messages(first, limit = 1))
        print(mes)
        print(result.dialogs[0].read_outbox_max_id)  # Мною отправленное письмо, прочитанное последнее
        print(result.dialogs[0])
        if result.dialogs[0].read_outbox_max_id >= mes:
            print('the message has been read')
        else:
            print('the message has not been read yet')

        # @client.on(telethon.events.NewMessage(chats=("CANNEL")))
        # async def handler(event):
        #     if 'user_id=2807' in str(event.message):
        #         if 'text' in str(event.message):
        #             await event.reply('MESSAGE')
        # # await client.send_message('@gexos', 'Вот это сообщение отправил от Python')
        user_id_list = []

        # response = client.wait_event(telethon.events.NewMessage(
        #     outgoing=True,
        #     forwards=None
        # ))
        # response = await response
        # code = response.message.message.strip()
        # print(code)

    client.loop.run_until_complete(run())
    def write():
        print('asdasd')
    client.add_event_handler(write, telethon.events.NewMessage(incoming=True, forwards=None))
    # client.add_event_handler(handler)
    return render(request, 'index.html')


#!/usr/bin/env python
import sys
from os import remove, path
import shutil

from setuptools import setup
from distutils.command.clean import clean
from distutils.extension import Extension

# if cythonize is not used - need to use Cython's build_ext
# so it autogenerates the C++ code before compiling
# look into cythonize as well - may require building libpolygon separately
from Cython.Distutils import build_ext
import versioneer

base = path.dirname(__file__)
pkg_dir = 'polygon'

cy_poly_dir = path.join(base, 'polygon')
lib_poly_dir = path.join(base, 'lib_polygon')

class cleandev(clean):
    description = ("cleans files generated by 'develop' mode and files "
                   "autogenerated by cython")

    def run(self):
        '''
        override base class to remove autogenerated cython stuff as well
        '''
        clean.run(self)
        dirs = ['build', pkg_dir + '.egg-info', 'dist']
        for d in dirs:
            try:
                d_full = path.join(base, d)
                shutil.rmtree(d_full)
                print('removed ' + d_full)
            except:
                pass

        # remove auto gen cython files
        auto_gen = ['cy_polygon.so', 'libpolygon.so', 'cy_polygon.cpp']
        for f in auto_gen:
            f_full = path.join(base, 'polygon', f)
            try:
                remove(f_full)
                print('removed ' + f_full)
            except:
                pass

# let's build the lib_polygons separately and link to it
#===============================================================================
# exts = [Extension("polygon.libpolygon",
#                   [path.join(lib_poly_dir, 'polygon.cpp')],
#                   language="c++",
#                   include_dirs=[os.path.join(base, 'lib_polygons')],
#                   )]
#===============================================================================

if sys.platform == 'win32':
    libs = []
else:
    libs = ['z', 'curl']

exts = []
exts.append(Extension("polygon.cy_polygon",
                      [path.join(lib_poly_dir, 'polygon.cpp'),
                       path.join(cy_poly_dir, 'cy_polygon.pyx')],
                      language="c++",
                      include_dirs=[path.join(base, 'polygon'),
                                    lib_poly_dir],
                      # add external libraries for testing only
                      libraries=libs,
                      ))

# tested on Mac OSX (10.10.3), with setuptools 17.1.1, py27_0
# need egg-info folder since package contains compiled code
cmd_class = {'build_ext': build_ext, 'clean': cleandev}
cmd_class.update(versioneer.get_cmdclass())
setup(name=pkg_dir,
      cmdclass=cmd_class,
      entry_points={'console_scripts': ['hello = polygon:hello_polygon']},
      version=versioneer.get_version(),
      description='introduce build error in demo package',
      author="Jasmine Sandhu",
      author_email="jsandhu@continuum.io",
      url="https://github.com/sandhujasmine/CythonExample",
      ext_modules=exts,
      packages=[pkg_dir],
      zip_safe=False)


from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.template.loader import render_to_string
from django.core.exceptions import ValidationError
from django.conf import settings

from cms.models import CMSPlugin

from geopy.geocoders import Here

def get_geolocation(street: str, postal_code: str, city: str) -> dict:
    geolocator = Here(settings.HERE_APP_ID, settings.HERE_APP_CODE)
    location = geolocator.geocode(" ".join([street, postal_code, city]))
    if location:
        return {"lat": location.latitude, "lng": location.longitude}
    return {"lat": None, "lng": None}

class Map(CMSPlugin):

    STYLE_CHOICES = (("google", "Google Maps"), ("leaflet", "Leaflet"))
    style = models.CharField(
        _("style"), max_length=25, choices=STYLE_CHOICES, default="leaflet"
    )

    leaflet_tile_url = models.CharField(
        _("tile url"),
        max_length=255,
        default="http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",
    )

    height = models.IntegerField(
        _("height"), help_text=_("height of the map in px."), default=400
    )
    ZOOM_LEVELS = map(lambda c: (c, str(c)), range(22))
    zoom = models.IntegerField(_("zoom"), choices=ZOOM_LEVELS, default=8)

    # center address
    street = models.CharField(
        _("street"),
        max_length=100,
        help_text=_("address for center of map"),
        null=True,
        blank=True,
    )
    postal_code = models.CharField(_("postal code"), max_length=10)
    city = models.CharField(_("city"), max_length=100)
    lat = models.DecimalField(
        _("lat"), null=True, blank=True, decimal_places=6, max_digits=10
    )
    lng = models.DecimalField(
        _("lng"), null=True, blank=True, decimal_places=6, max_digits=10
    )

    def clean(self, *args, **kwargs):
        if self.style == "mapbox" and self.mapbox_access_token == "":
            raise ValidationError(
                {"mapbox_access_token": _("mapbox access token is required")}
            )
        if self.style == "mapbox" and self.mapbox_map_id == "":
            raise ValidationError({"mapbox_map_id": _("mapbox map id is required")})

        try:
            location = get_geolocation(self.street or "", self.postal_code, self.city)
            self.lat = location["lat"]
            self.lng = location["lng"]
        except:
            raise ValidationError(
                {
                    "street": _("not a valid address"),
                    "postal_code": _("not a valid address"),
                    "city": _("not a valid address"),
                }
            )

    def copy_relations(self, oldinstance):
        for pin in oldinstance.pins.all():
            pin.pk = None
            pin.map_plugin = self
            pin.save()

class Pin(models.Model):
    name = models.CharField(_("name"), max_length=50)
    map_plugin = models.ForeignKey(Map, related_name="pins")

    street = models.CharField(_("street"), max_length=100, null=True, blank=True)
    postal_code = models.CharField(_("postal code"), max_length=10)
    city = models.CharField(_("city"), max_length=100)

    link = models.URLField(_("link"), blank=True, null=True)
    link_title = models.CharField(
        _("link title"), max_length=255, blank=True, null=True
    )

    description = models.TextField(_("description"), blank=True, null=True)

    COLOR_CHOICES = (
        ("redIcon", _("red")),
        ("blueIcon", _("blue")),
        ("greenIcon", _("green")),
        ("yellowIcon", _("yellow")),
    )

    pin_color = models.CharField(max_length=20, choices=COLOR_CHOICES)

    lat = models.DecimalField(
        _("lat"), null=True, blank=True, decimal_places=6, max_digits=10
    )
    lng = models.DecimalField(
        _("lng"), null=True, blank=True, decimal_places=6, max_digits=10
    )

    @property
    def infowindow(self):
        context = {}
        context["name"] = self.name
        context["street"] = self.street
        context["postal_code"] = self.postal_code
        context["city"] = self.city
        context["link"] = self.link
        context["link_title"] = self.link_title
        context["description"] = self.description

        return render_to_string(
            "cmsplugin_multipinmap/infowindow.html", context
        ).replace("\n", "")

    def clean(self, *args, **kwargs):
        try:
            location = get_geolocation(self.street or "", self.postal_code, self.city)
            self.lat = location["lat"]
            self.lng = location["lng"]
        except:
            raise ValidationError(
                {
                    "street": _("not a valid address"),
                    "postal_code": _("not a valid address"),
                    "city": _("not a valid address"),
                }
            )

    def __str__(self):
        return self.name