import json
from builtins import list
from datetime import datetime
import bisect
import copy
import numpy as np
import os
from os import listdir
from os.path import isfile, join
import MarketStateManager
import TransactionBasics

PeakFeatureCount = TransactionBasics.PeakFeatureCount
MaxMinDataLen = 8
UpSideDataLen = 6
TotalExtraFeatureCount = PeakFeatureCount + MaxMinDataLen + UpSideDataLen
percent = 0.01

class BuySellHandler:
    def __init__(self, jsonIn, transactionParam,marketState):
        self.marketState = marketState
        self.buyTimeInSeconds = 0
        self.buyPrice = 0.0
        self.sellPrice = 0.0
        self.sellTimeInSeconds = 0

        self.transactions = []
        self.currencyName = ""
        self.transactionParam = transactionParam
        self.mustSellList = []
        self.keepList = []
        self.__Parse(jsonIn)

        self.lowestTransaction = TransactionBasics.TransactionCountPerSecBase/3
        self.acceptedTransLimit = TransactionBasics.TransactionLimitPerSecBase/3
        self.dataList = []

        tempTransaction = json.loads(jsonIn["transactions"])
        if len(tempTransaction) < 60:
            return
        prices = list(map(lambda x: float(x["p"]), tempTransaction))

        self.peakIndex = prices.index(max(prices))
        self.peakTime = int(tempTransaction[self.peakIndex]["T"])//1000
        self.peakVal = float(tempTransaction[self.peakIndex]["p"])

        self.minPeakIndex = prices.index(min(prices))
        self.minPeakTime = int(tempTransaction[self.minPeakIndex]["T"])//1000
        self.minPeakVal = float(tempTransaction[self.minPeakIndex]["p"])

        self.__DivideDataInSeconds(tempTransaction) #populates the dataList with TransactionData
        self.__AppendToPatternList() # deletes dataList and populates mustBuyList, patternList badPatternList

    def __Parse(self, jsonIn):

        epoch = datetime.utcfromtimestamp(0)
        self.buyPrice = float(jsonIn["buyPrice"])
        datetime_object = datetime.strptime(jsonIn["buyTime"].split(".")[0], '%Y-%b-%d %H:%M:%S')
        self.buyTimeInSeconds = (datetime_object - epoch).total_seconds()
        self.sellPrice = float(jsonIn["sellPrice"])
        datetime_object = datetime.strptime(jsonIn["sellTime"].split(".")[0], '%Y-%b-%d %H:%M:%S')
        self.sellTimeInSeconds = (datetime_object - epoch).total_seconds()
        self.currencyName = jsonIn["name"]

    def __DivideDataInSeconds(self, jsonIn):
        transactionData = TransactionBasics.TransactionData()
        lastEndTime = 0
        stopMiliSecs = int(jsonIn[-1]["T"])
        for x in range(len(jsonIn)):
            curElement = jsonIn[x]
            curMiliSecs = int(curElement["T"])
            if x == 0:
                lastEndTime = curMiliSecs + self.transactionParam.msec

            if curMiliSecs > lastEndTime:
                copyData = copy.deepcopy(transactionData)
                self.dataList.append(copyData)
                transactionData.Reset()
                transactionData.AddData(curElement)
                transactionData.SetTime(curMiliSecs // 1000)
                lastEndTime += self.transactionParam.msec
                while True:
                    if curMiliSecs > lastEndTime and lastEndTime < stopMiliSecs:
                        lastEndTime += self.transactionParam.msec
                        emptyData = TransactionBasics.TransactionData()
                        emptyData.SetTime(lastEndTime // 1000)
                        emptyData.lastPrice = copyData.lastPrice
                        self.dataList.append(emptyData)
                    else:
                        break
            else:
                transactionData.AddData(curElement)
        copyData = copy.deepcopy(transactionData)
        self.dataList.append(copyData)

    def __AppendToPatternList(self):
        lenArray = len(self.dataList)
        if lenArray == 0:
            return
        # print(lenArray, self.dataList)
        for x in range(0, lenArray):
            self.__AppendToPatternListImpl(self.transactionParam.gramCount, x, lenArray)
        del self.dataList

    def __AppendToPatternListImpl(self, ngramCount, curIndex, lenArray):
        totalCount = TransactionBasics.GetTotalPatternCount(ngramCount)
        startBin = curIndex + 1 - totalCount
        endBin = curIndex + 1
        if startBin < 0 or endBin > lenArray:
            return

        lastTotalTradePower = self.dataList[curIndex].totalBuy + self.dataList[curIndex].totalSell
        if lastTotalTradePower < self.acceptedTransLimit:
            return

        pattern = TransactionBasics.TransactionPattern()
        copyList = copy.deepcopy(self.dataList[startBin:endBin])
        dataRange = TransactionBasics.ReduceToNGrams(copyList, ngramCount)
        pattern.AppendWithOutPeaks( dataRange, self.marketState, self.buyPrice, self.buyTimeInSeconds)

        if self.__GetCategory(curIndex) == 1:
            self.mustSellList.append(copy.deepcopy(pattern))
        elif self.__GetCategory(curIndex) == 2:
            self.keepList.append(copy.deepcopy(pattern))

    def __GetCategory(self, curIndex):
        price = self.dataList[curIndex].maxPrice
        time = self.dataList[curIndex].timeInSecs

        if price > self.peakVal * 0.997:
            return 1  # Sell
        elif time > self.peakTime and price > self.peakVal * 0.995:
            return 1  # Sell
        elif time > self.buyTimeInSeconds and time < self.peakTime:
            return 2
        return -1

class BuyAnalyzeMerger:
    def __init__(self, transactionParam, marketState):
        self.mustSellList = []
        self.keepList = []

        self.handlerList = []
        self.peakHelperList = []
        self.transactionParam = transactionParam
        self.marketState = marketState

    def AddFile(self, jsonIn):
        for index in range(len(jsonIn)):
            if not jsonIn[index]:
                continue

            jsonPeakTrans = jsonIn[index]
            handler = BuySellHandler(jsonPeakTrans,self.transactionParam,self.marketState)
            self.handlerList.append(handler)

    def Finalize(self):
        for peak in self.handlerList:
            self.__MergeInTransactions(peak)
            del peak

    def toSellTransactions(self):
        mustSellCount = len(self.mustSellList)
        keepCount = len(self.keepList)
        #self.Print()
        print("Must sell count: ", mustSellCount, " Keep count: ", keepCount)

        allData = np.concatenate( (self.mustSellList, self.keepList), axis=0)
        print("Must sell count: ", mustSellCount, " Keep count: ", keepCount)
        return allData

    def toSellResultsNumpy(self):
        mustSellCount = len(self.mustSellList)
        keepCount = len(self.keepList)

        print("Must sell count: ", mustSellCount, " Keep count: ", keepCount)
        mustSellResult = [1] * mustSellCount
        keepResult  = [0] * keepCount
        returnPatternList = mustSellResult + keepResult
        return returnPatternList

    def Print(self):

        #mustBuyList = np.array(self.mustBuyList)
        buyList = np.array(self.mustSellList)
        badList = np.array(self.keepList)

        for i in range(len(self.patternList[0])):
            a = {'Good': buyList[:, -(i + 1)],
                 'Bad': badList[:, -(i + 1)]}
            #df = pd.DataFrame.from_dict( a, orient='index')
            #df = df.transpose()
            #df.plot.box()
            #mustBuyLegend = str(np.quantile(mustBuyList[:, -(i + 1)], 0.1)) + "," + str(np.quantile(mustBuyList[:, -(i + 1)], 0.5)) + "," + str(np.quantile(mustBuyList[:, -(i + 1)], 0.9))
            buyLegend = str(np.quantile(buyList[:, -(i + 1)], 0.1)) + "," + str(np.quantile(buyList[:, -(i + 1)], 0.25)) + "," \
                        + str(np.quantile(buyList[:, -(i + 1)], 0.5)) + "," + str(np.quantile(buyList[:, -(i + 1)], 0.75)) + "," + str(np.quantile(buyList[:, -(i + 1)], 0.9))
            badLegend = str(np.quantile(badList[:, -(i + 1)], 0.1)) + "," + str(np.quantile(badList[:, -(i + 1)], 0.25)) +\
                        "," + str(np.quantile(badList[:, -(i + 1)], 0.5)) + "," + str(np.quantile(badList[:, -(i + 1)], 0.75)) + "," + str(np.quantile(badList[:, -(i + 1)], 0.9))
            print(str(self.transactionParam.msec) ,"_" , str(i), "_" , buyLegend , " ", badLegend)
            #plt.savefig('Plots/' + str(self.transactionParam.msec) + "_" + str(i) + "_box.pdf")
            #plt.cla()
            #plt.clf()

        #plt.close()

    def __MergeInTransactions(self, handler):

        for pattern in handler.mustSellList:
            self.mustSellList.append(pattern.GetFeatures())

        for pattern in handler.keepList:
            self.keepList.append(pattern.GetFeatures())

class BuyAnalyzeManager:

    def __init__(self, transactionParamList, marketStateIn):
        self.marketState = marketStateIn
        self.transParamList = transactionParamList
        self.buySellMergerList = []
        self.CreateBuyAnalyzeMergers()
        print(self.buySellMergerList)
        self.BuyAnalyzeMergers()
        self.FinalizeMergers()

    def BuyAnalyzeMergers(self):
        jumpDataFolderPath = os.path.abspath(os.getcwd()) + "/Data/BuySellData/"
        onlyJumpFiles = [f for f in listdir(jumpDataFolderPath) if isfile(join(jumpDataFolderPath, f))]
        for fileName in onlyJumpFiles:
            print("Reading Buy", jumpDataFolderPath + fileName, " ")
            file = open(jumpDataFolderPath + fileName, "r")
            try:
                jsonDictionary = json.load(file)
                for merger in self.buySellMergerList:
                    merger.AddFile(jsonDictionary)
            except:
                print("There was a exception in ", fileName)

    def toSellTransactions(self, index):
        return self.buySellMergerList[index].toSellTransactions()

    def toSellResultsNumpy(self, index):
        return self.buySellMergerList[index].toSellResultsNumpy()

    def FinalizeMergers(self):
        for transactionIndex in range(len(self.transParamList)):
            self.buySellMergerList[transactionIndex].Finalize()

    def CreateBuyAnalyzeMergers(self):
        for transactionIndex in range(len(self.transParamList)):
            newMerger = BuyAnalyzeMerger(self.transParamList[transactionIndex], self.marketState)
            self.buySellMergerList.append(newMerger)


from kivy.app import App
from kivy.uix.textinput import TextInput
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.button import Button
from kivy.uix.label import Label

class MyTI(TextInput):
	multiline = False
	def on_text_validate(self):
		print(self.text)
		lbl = self.parent.parent.lbl
		lbl.text = self.text

class Color(Button):
    def on_press(self):
        lbl = self.parent.parent.lbl
        if str(lbl.color) == str([1,1,1,1]):
            lbl.color = (0,0,1,1)
        elif str(lbl.color) == str([0,0,1,1]):
            lbl.color = (1,0,0,1)
        elif str(lbl.color) == str([1,0,0,1]):
            lbl.color = (1,1,1,1)
        else:
            lbl.text = str(lbl.color)

class MyRoot(BoxLayout):
    def __init__(self,**kwargs):
        super().__init__(**kwargs)
        box = BoxLayout(orientation="vertical")
        tex = MyTI(text="Please Enter")
        self.lbl = Label(text="",color=[1,1,1,1])
        btn = Color(text="Change the text color")
        box.add_widget(tex)
        box.add_widget(self.lbl)
        box.add_widget(btn)
        self.add_widget(box)

class Textmaker(App):
	def build(self):
		return MyRoot()

Textmaker().run()


from flask import Flask
import json

app = Flask(__name__)

pais = [
    {
        'id': 1,
        'name': "Mexico",
        "description": "No Mexico falamos espanhol"
    },
    {
        "id": 2,
        "name": "Brasil",
        "description": "Aqui é Br"
    },
    {
        "id": 3,
        "name": "Italia",
        "description": "Comemos Pizaa"
    },{
        "id": 4,
        "name": "França",
        "description": "Comemos Pao"
    },
]

paisJSON = json.dumps(pais)

@app.route('/')
def home():
    return "paises",200

@app.route('/api/pais', methods=["GET"])
def consultar():
    return paisJSON, 204

if __name__ == '__main__':
    app.run()


import os
import math
import random

# 화면을 클리어하는 함수
def clear_screen():
    os.system('cls')
    return

# 플레이어에게 계단의 개수를 입력 받은 값을 바탕으로 계단을 만드는 함수
def create_stair(row, col):
    s_matrix = [[" " for i in range(col)] for j in range(row)]
    s_matrix[0][0] = "○"
    s_matrix[0][col - 1] = "●"
    space = col - 1

    for i in range(1, row):
        for j in range(i):
            s_matrix[i][j] = "▨"

        for k in range(space + i - 1, col):
            s_matrix[i][k] = "▨"
        space -= 2

    return s_matrix

# 묵찌빠 결과에 따라 계단을 업데이트하는 함수
def update_stair(s_matrix, winner, step):
    global p_row_prev
    global p_col_prev
    global c_row_prev
    global c_col_prev

    # 먼저 전에 있던 플레이어와 컴퓨터의 위치를 빈 칸으로 초기화한다
    s_matrix[p_row_prev][p_col_prev] = " "
    s_matrix[c_row_prev][c_col_prev] = " "

    # 플레이어와 컴퓨터 중 승자를 확인하고, step의 값에 따라 승자가 이동해야 하는 칸을 각 row와 col에 업데이트 한다
    if winner == "플레이어":                                # 플레이어가 묵찌빠를 이겼을 때
        if num_stairs % 2 == 0:                           # 계단 개수가 짝수일 때
            if p_col_prev + step <= COL // 2:
                p_row_prev += step
                p_col_prev += step
            else:
                p_row_prev = COL - (p_col_prev + step) - 1
                p_col_prev += step
        else:                                             # 계단 개수가 홀수일 때
            if p_col_prev + step < COL // 2:              # 홀수일 때는 등호가 없음
                p_row_prev += step
                p_col_prev += step
            else:
                p_row_prev = COL - (p_col_prev + step) - 1
                p_col_prev += step
    else:                                                 # 컴퓨터가 묵찌빠를 이겼을 때
        if c_col_prev - step >= COL // 2:
            c_row_prev += step
            c_col_prev -= step
        else:
            c_row_prev = c_col_prev - step
            c_col_prev -= step

    # 업데이트한 플레이어와 컴퓨터의 row와 col을 바탕으로 s_matrix에 그림값을 넣어준다.
    if p_row_prev == c_row_prev and p_col_prev == c_col_prev:               # 플레이어와 컴퓨터가 같은 위치에 있는 상황
        s_matrix[p_row_prev][p_col_prev] = "◑"
    else:     # 플레이어나 컴퓨터가 승리 했을 때 s_matrix의 값을 벗어나는 경우가 있기 때문에 예외 처리를 하면서 각자의 위치에 그림을 넣어준다.
        if p_col_prev >= COL:
            p_col_prev = COL - 1
            p_row_prev = 0
        if c_col_prev < 0:
            c_col_prev = 0
            c_row_prev = 0
        s_matrix[p_row_prev][p_col_prev] = "○"
        s_matrix[c_row_prev][c_col_prev] = "●"

# 화면에 계단과 플레이어들을 출력하는 함수
def print_stairs(s_matrix):

    # 이중 for문을 사용하여 계단 메트릭스 출력
    for row in s_matrix:
        for e in row:
            print(e, end=" ")
        print()

# 화면에 게임 현황과 계단을 보여주는 함수
def print_display():
    clear_screen()
    print(f"총 계단 수: {num_stairs}")
    print(f"PLAYER:   ○  <{min(player_step, num_stairs):2d}>")
    print(f"COMPUTER: ●  <{min(com_step, num_stairs):2d}>")
    print_stairs(S_MATRIX)

# random 모듈을 활용하여 무작위로 가위, 바위, 보 중 하나를 반환하는 함수
def computer_choice():
    return random.choice(["가위", "바위", "보"])

# 플레이어가 올바른 입력값을 입력할 때까지 input을 받는 함수
def player_choice():
    player = ""
    while player not in ["가위", "바위", "보"]:
        player = input("가위, 바위, 보 중 하나 선택:")
    return player

# 플레이어와 컴퓨터의 가위바위보 결과가 돌려주는 함수
# 0이면 플레이어 승리, 1이면 무승부, 2이면 컴퓨터 승리
def check_result(p_choice, c_choice):
    if p_choice == c_choice:                  # 플레이어와 컴퓨터가 같은 것을 냈을 때
        return 1
    if p_choice == rock:
        if c_choice == paper:
            return 2
        return 0
    elif p_choice == paper:
        if c_choice == scissor:
            return 2
        return 0
    else:
        if c_choice == rock:
            return 2
        return 0

# 가위바위보 모양은 https://gist.github.com/wynand1004/b5c521ea8392e9c6bfe101b025c39abe 를 참조함

# 화면에 바위를 출력하는 함수
def print_rock():
    print("""
        _______
    ---'   ____)
          (_____)
          (_____)
          (____)
    ---.__(___)
    """)

# 화면에 보를 출력하는 함수
def print_paper():
    print("""
         _______
    ---'    ____)____
               ______)
              ________)
             _______)
    ---.__________)
    """)

# 화면에 가위를 출력하는 함수
def print_scissors():
    print("""
        _______
    ---'   ____)____
              ______)
           __________)
          (____)
    ---.__(___)
    """)

if __name__ == "__main__":

    rock = "바위"
    paper = "보"
    scissor = "가위"
    num_stairs = 0

    print("======================")
    print("[묵찌빠 계단 오르기]")
    print("======================")
    start_img = create_stair(7, 11)
    print_stairs(start_img)
    print()

    while num_stairs < 10 or num_stairs > 30:
        num_stairs = int(input("게임을 위한 계단의 개수를 입력해주세요. <10 ~ 30> >> "))

    # 화면 초기화
    clear_screen()

    # 입력된 계단의 수에서 row와 col을 계산한다
    ROW = math.ceil(num_stairs / 2) + 1
    COL = num_stairs + 1

    # player 값들 초기화
    p_row = 0
    p_col = 0
    player_step = 0

    # computer 값들 초기화
    c_row = 0
    c_col = COL
    com_step = 0

    # row와 col을 이용해서 stair matrix를 만든다
    S_MATRIX = create_stair(ROW, COL)

    p_row_prev = p_row
    p_col_prev = p_col
    c_row_prev = c_row
    c_col_prev = c_col - 1

    while True:
        # stair matrix를 출력한다
        print_display()
        input("\n계속하려면 엔터를 눌러주세요...")

        ################## 공격권 결정 가위바위보 #######################
        turn = ""     # 공격권
        while True:
            clear_screen()
            print(["공격권 결정 가위바위보"])

            # 이동 칸 수 초기화
            step = 1

            # 플레이어가 올바른 입력값을 입력할 때까지 input을 받는다
            player = player_choice()

            # 컴퓨터 무작위로 가위바위보 중 하나 선택
            com = computer_choice()

            # 컴퓨터 선택 출력
            print("\n[컴퓨터 선택]")
            if com == rock:
                print_rock()
            elif com == paper:
                print_paper()
            else:
                print_scissors()

            # 플레이어 선택 출력
            print("\n[플레이어 선택]")
            if player == rock:
                print_rock()
            elif player == paper:
                print_paper()
            else:
                print_scissors()

            # 게임 결과를 계산하고 result에 담는다
            result = check_result(player, com)
            if result == 0:
                turn = "player"
                print("[결과] 플레이어 공격, 컴퓨터 수비입니다.")
                input("\n계속하려면 엔터를 눌러주세요...")
                break
            elif result == 2:
                turn = "com"
                print("[결과] 컴퓨터 공격, 플레이어 수비입니다.")
                input("\n계속하려면 엔터를 눌러주세요...")
                break
            else:
                print("[결과] 무승부입니다")
                input("\n계속하려면 엔터를 눌러주세요...")

        ###################################################################

        ########################### 묵찌빠 #################################
        while True:
            clear_screen()
            print("[묵찌빠]")
            print(f"승리 시 이동 칸 수: {step}")

            if turn == "player":
                print("플레이어 공격, 컴퓨터 수비입니다.")
            else:
                print("컴퓨터 공격, 플레이어 수비입니다.")

            # 플레이어가 올바른 입력값을 입력할 때까지 input을 받는다
            player = player_choice()

            # 컴퓨터 무작위로 가위바위보 중 하나 선택
            com = computer_choice()

            # 컴퓨터 선택 출력
            print("\n[컴퓨터 선택]")
            if com == rock:
                print_rock()
            elif com == paper:
                print_paper()
            else:
                print_scissors()

            # 플레이어 선택 출력
            print("[플레이어 선택]")
            if player == rock:
                print_rock()
            elif player == paper:
                print_paper()
            else:
                print_scissors()

            # 게임 결과를 계산하고 result에 담는다
            result = check_result(player, com)
            if result == 0:
                turn = "player"
                print("[결과] 플레이어 공격, 컴퓨터 수비입니다.")
            elif result == 2:
                turn = "com"
                print("[결과] 컴퓨터 공격, 플레이어 수비입니다.")
            else:
                print("[결과] 묵찌빠 종료")
                winner = "플레이어" if turn == "player" else "컴퓨터"

                update_stair(S_MATRIX, winner, step)
                if winner == "플레이어":
                    player_step += step
                else:
                    com_step += step

                print(f"{winner} 승, {step}칸 이동합니다")
                input("\n계속하려면 엔터를 눌러주세요...")
                clear_screen()
                break

            # 승부가 결정나지 않아 이동 칸 수를 하나 증가시킨다
            step += 1
            input("\n계속하려면 엔터를 눌러주세요...")
        ###################################################################

        # 승자가 있는지 확인하는 if문
        if player_step >= num_stairs:
            print_display()
            print()
            print("▨ " * COL)
            print(" " * (num_stairs - 10) + "플레이어 최종 승리!!!")
            print("▨ " * COL)
            print("\n게임을 종료합니다...")
            break
        elif com_step >= num_stairs:
            print_display()
            print()
            print("▨ " * COL)
            print(" " * (num_stairs - 9) + "컴퓨터 최종 승리!!!")
            print("▨ " * COL)
            print("\n게임을 종료합니다...")
            break


from requests import get
from bs4 import BeautifulSoup
import pandas as pd
import os

def get_one_blog_article(url):
    '''This function takes in a url in string format, it provides a User-Agent to the URL and requests a response using the URL and headers.
    It then uses the BeautifulSoup library to parse the URL contents. Lastly, it creates an empty dictionary, then stores the article title,
    date published,and blog article contents in the dictionary. It returns the dictionary as output.
    '''

    headers = {'User-Agent': 'Innis Data Science cohort'}
    response = get(url, headers=headers)

    # Make a soup variable holding the response content
    soup = BeautifulSoup(response.text, 'html.parser')

    output = {}
    output['title'] = soup.select_one('h1.entry-title').text
    output['published'] = soup.select_one('.published').text
    output['contents'] = soup.select_one('div.entry-content').text

    return output

urls = [
    'https://codeup.com/codeup-news/codeup-start-dates-for-march-2022/',
    'https://codeup.com/featured/5-books-every-woman-in-tech-should-read/',
    'https://codeup.com/codeup-news/dallas-campus-re-opens-with-new-grant-partner/',
    'https://codeup.com/codeup-news/codeups-placement-team-continues-setting-records/',
    'https://codeup.com/it-training/it-certifications-101/'
]

def get_blog_articles(urls):
    '''This function takes in a list of urls in string format.
    It applies the get_one_blog_article function above to obtain the title and contents
    of each of the blog urls. It then returns a dataframe with the title and contents.
    '''

    # Create a list of dictionaries
    list_dic = [get_one_blog_article(url) for url in urls]
    # Return a data frame
    return pd.DataFrame(list_dic)   #.style.set_properties(**{'text-align':'left'})

# Take in ANY card
def parse_news_card(card, category):
    output = {}
    output['category'] = category
    output['title'] = card.select_one('a.clickable').text.strip()

    card_content = card.select_one('div.news-card-content')
    output['content'] = card_content.select_one('div').text
    output['author'] = card_content.select_one('.author').text
    output['published'] = card_content.select_one('.time').attrs['content']

    return output

# Need a function to obtain ANY CATEGORY
# take in a category and returns a list of dictionaries list[dict]
def parse_inshorts_page(category):
    url = 'https://inshorts.com/en/read/' + category
    response = get(url)
    soup = BeautifulSoup(response.text)
    articles = [parse_news_card(card, category) for card in soup.select('.news-card')]

    return articles

def get_news_articles():
    categories = ['business', 'sports', 'entertainment', 'technology']
    articles = []

    for category in categories:
        print(f'Getting articles for {category}')
        category_articles = parse_inshorts_page(category)
        articles.extend(category_articles)
    return pd.DataFrame(articles)


import time
import sys

import chess

from util import fen4, move_list_to_sans

import evaluate

from engine import Engine, SearchStats

import cProfile

class Game:
    def __init__(self, board = chess.Board(), config_b = {"time-limit-s": 120}, config_w = {"time-limit-s": 60}):
        self.board = board
        # TODO - docs reckon this only copies the base-board - what about previous moves?
        self.engines = [Engine(board.copy(), config_b), Engine(board.copy(), config_w)]

    def play(self):
        total_engine_time_s = 0
        print_board = True

        w_engine = self.engines[chess.WHITE]
        b_engine = self.engines[chess.BLACK]

        while True:
            legal_move_sans = [self.board.san(m) for m in self.board.legal_moves]

            print()

            if print_board:
                print(self.board.fen())
                # print()
                # print(self.board)
                print()
                print(self.board.unicode(invert_color=True, empty_square='.')) # Unicode chars seems backwards; Don't seem to have the default empty_square unicode in my font
                print()
            print_board = True

            if self.board.is_game_over():

                reason = 'unknown'
                if self.board.is_checkmate():
                    reason = 'checkmate'
                elif self.board.is_stalemate():
                    reason = 'stalemate'
                elif self.board.is_insufficient_material():
                    reason = 'insufficient material'
                elif self.board.is_seventyfive_moves():
                    reason = '75 moves played without progress'
                elif self.board.is_fivefold_repetition():
                    reason = '5-fold repetition'

                print("Game over: %s (%s)" % (self.board.result(), reason))

                print()
                print("Moves: %s" % move_list_to_sans(self.board.root(), self.board.move_stack))
                return

            print("%s to move" % ["Black", "White"][self.board.turn])
            print()
            w_val = w_engine.static_eval()
            b_val = b_engine.static_eval()
            print("Static eval   - positive is White advantage: white engine %d cp black engine %d cp" % (w_val, b_val))
            sign = [-1, 1][w_engine.board.turn]
            w_qval = w_engine.quiesce_alphabeta(SearchStats(1, w_engine.MAX_QDEPTH), 0, w_val * sign) * sign
            b_qval = b_engine.quiesce_alphabeta(SearchStats(1, b_engine.MAX_QDEPTH), 0, b_val * sign) * sign
            print("Quiesced eval - positive is White advantage: white engine %d cp black engine %d cp" % (w_qval, b_qval))
            print()
            print("Legal moves: %s" % " ".join(legal_move_sans))
            print()
            move_san = input("Enter move - SAN format - or 'pass' or 'engine': ")
            print()

            if move_san == "pass":
                # null move - let the other side move
                if self.board.is_check():
                    print()
                    print(">>> Illegal to 'pass' when in check <<<")
                    print_board = False
                    continue

                move = chess.Move.null()

            elif move_san == 'engine':
                engine = self.engines[self.board.turn]
                engine_move, val, pv, stats = engine.gen_move()

                move = engine_move

            else:
                if not move_san in legal_move_sans:
                    print()
                    print(">>> Illegal move \"%s\" entered <<<" % move_san)
                    print_board = False
                    continue

                move = self.board.parse_san(move_san)

            self.board.push(move)
            w_engine.make_move(move)
            b_engine.make_move(move)

def main():
    print("Hallo RPJ - let's play chess")
    # game = Game(chess.Board(), 3*60)
    game = Game(chess.Board())
    # game = Game(chess.Board("2r1r1k1/2bn4/R1p1p3/2P2p1p/1P3P2/3B2PP/8/2BR2K1 w - - 1 32"))
    # game = Game(chess.Board("r1bqk2r/ppp2ppp/2nbpn2/3p4/3P4/2N1PN2/PPP1BPPP/R1BQK2R w KQkq - 2 6"))
    game.play()

if __name__ == "__main__":
    main()
    # cProfile.run("main()")


"""
Given an integer array nums, in which exactly two elements appear only once and all the other elements appear exactly twice. Find the two elements that appear only once. You can return the answer in any order.

You must write an algorithm that runs in linear runtime complexity and uses only constant extra space.

Example 1:

Input: nums = [1,2,1,3,2,5]
Output: [3,5]
Explanation:  [5, 3] is also a valid answer.
Example 2:

Input: nums = [-1,0]
Output: [-1,0]
Example 3:

Input: nums = [0,1]
Output: [1,0]

Constraints:

2 <= nums.length <= 3 * 104
-231 <= nums[i] <= 231 - 1
Each integer in nums will appear twice, only two integers will appear once.
"""
from typing import List

class SingleNumberIII:
    def singleNumber(self, nums: List[int]) -> List[int]:
        ans = []
        idx = 0
        nums = sorted(nums)
        i = 0
        while i < len(nums):
            if i == len(nums) - 1:
                ans.append(nums[i])
                break
            else:
                if nums[i] != nums[i+1]:
                    ans.append(nums[i])
                    idx += 1
                    if(idx == 2):
                        break
                    else:
                        i -= 1
            i += 2

        return ans


from ml_collections.config_dict import ConfigDict
import inspect
from e3nn import o3
import torch
from torch_runstats.scatter import scatter
import math
import numpy as np

def setSeed(seed):
  torch.manual_seed(seed)
  np.random.seed(seed)
  torch.cuda.manual_seed_all(seed)
  torch.backends.cudnn.deterministic = True

def getScaler(operations):
  """
  This function normalizes data by scaling and shifting.
  Example args: [('N', ('shift', 'CA', '-1')), ('CA', ('scale', 25))]
  """
  def scaler(batch):
    batch = batch.clone()
    node_segment = batch.nodeSegment()
    for ops in operations:
      key, op = ops
      if op[0] == 'scale':
        if isinstance(key, tuple) or isinstance(key, list):
          for tmp in key:
            batch[tmp] = batch[tmp]*op[1]
        else:
          batch[key] = batch[key]*op[1]
      elif op[0] == 'shift':
        if op[1] == 'mean':
          n_nodes = batch['_n_nodes'].view(-1, 1)
          center = scatter(batch[key], node_segment, dim=0, reduce='sum')
          center = center/n_nodes
          batch[key] = batch[key] - center[node_segment]
        elif op[1] in batch:
          sign = 1
          if len(op) == 3:
            sign = op[2]
          batch[key] = batch[key] + sign * batch[op[1]]
        else:
          raise ValueError()
      else:
        raise ValueError()
    return batch
  return scaler

def insertAfter(lst, key, item):
    for i, layer in enumerate(lst):
        if layer[0] == key:
            lst = lst[:i+1] + [item] + lst[i+1:]
            return lst
    raise ValueError(f'Key {key} not found.')

def replace(lst, key, item):
    for i, layer in enumerate(lst):
        if layer[0] == key:
            lst = lst[:i] + [item] + lst[i+1:]
            return lst
    raise ValueError(f'Key {key} not found.')

def transformMatrix(irreps_l, irreps_r, mat, H):
    D_l = irreps_l.D_from_matrix(mat)
    D_r = irreps_r.D_from_matrix(mat)
    return D_l @ H @ D_r.transpose(-1, -2)

def tanhlu(x):
    return torch.tanh(x) * torch.abs(x)

@torch.jit.script
def ShiftedSoftPlus(x):
    return torch.nn.functional.softplus(x) - math.log(2.0)

activations = {
    "abs": torch.abs,
    "tanh": torch.tanh,
    "ssp": ShiftedSoftPlus,
    "silu": torch.nn.functional.silu,
    "tanhlu": tanhlu,
}

def tp_path_exists(irreps_in1, irreps_in2, ir_out):
    irreps_in1 = o3.Irreps(irreps_in1).simplify()
    irreps_in2 = o3.Irreps(irreps_in2).simplify()
    ir_out = o3.Irrep(ir_out)

    for _, ir1 in irreps_in1:
        for _, ir2 in irreps_in2:
            if ir_out in ir1 * ir2:
                return True
    return False

def build(node, **kwargs):
    """Instantiates a layer using its config node"""
    if isinstance(node, dict) or isinstance(node, ConfigDict):
        func = node["module"]
        kwargs.update(**node)

    elif isinstance(node, list) or isinstance(node, tuple):
        func = node[0]
        args = node[1:]

    else:
        func = node

    args = []
    if "module" in kwargs:
        kwargs.pop("module")
    kwargs = pruneArgs(func, **kwargs)
    return func(*args, **kwargs)

def pruneArgs(_func=None, prefix="", **kwargs):
    if not prefix == "":
        args = {}
        for key, value in kwargs.items():
            if key.startswith(prefix):
                key = key[len(prefix) + 1 :]
                args[key] = value
    else:
        args = kwargs

    if not _func is None:
        arg_spec = inspect.getfullargspec(_func)
        if arg_spec.varkw:
            return args
        else:
            pnames = inspect.signature(_func).parameters
            return {key: args[key] for key in args if key in pnames}
    return args

def keyMap(dic, key_mapping):
    if isinstance(dic, dict):
        result = {}
        for key, value in dic.items():
            if key in key_mapping:
                new_key = key_mapping[key]
                if isinstance(new_key, str):
                    result[new_key] = value
                else:
                    for item in new_key:
                        result[item] = value
            else:
                result[key] = value
        return result
    else:  # an instance of data.Data
        attrs = keyMap(dic.attrs, key_mapping)
        data = keyMap(dic.data, key_mapping)
        return type(dic)(attrs, **data)

def _countParameters(module):
    return sum([param.numel() for param in module.parameters() if param.requires_grad])

def countParameters(model):
    for name, module in model.named_modules():
        cnt = _countParameters(module)
        if cnt > 0:
            print(name, cnt)


# an array list
# Sarah A. and Stefany A.

gluten_ls = dict([["flour","black bean puree"],["breadcrumbs","ground flaxseed or crushed potato chips"],["flour tortillas","corn tortillas"]])
sugar_ls = dict([["chocolate chips","cacao nibs"]])
dairy_ls = dict([["butter","avocado puree or olive oil"],["milk","rice, soy, oat, or almond milk"],["heavy cream","coconut milk"],["cheese","soy cheese"],["cream cheese","dairy-free cream cheese"],["sour cream","dairy-free sour cream"]])
vegan_ls = dict([["butter","avocado puree or olive oil"],["milk","almond milk"], ["heavy cream","coconut milk"],["ground beef","textured vegetable protein"]])
lowso_ls = dict([["salt","herbs, citrus juice, or garlic powder"],["breadcrumbs","rolled oats"]])
lgi_ls = dict([["rice","wild rice"],["penne","cheese tortellini"],["white bread","sourdough bread"]])
ibs_ls = dict([["milk","rice, soy, or oat milk"],["cheese","soy cheese"],["cream cheese","dairy-free cream cheese"],["sour cream","dairy-free sour cream"],["ground beef","textured vegetable protein"],["mayonnaise","fat-free mayonnaise"],["salad dressing","fat-free salad dressing"]])

    #join the sentence so that only one variable is edited
    #sentc.join

def recur_replace(key, file, dictionary_lst):
    if key == len(dictionary_lst):
        return "Done"
    else:
        value_ls=[]
        for val in dictionary_lst.itervalues():
            value_ls.append(val)
        for i in value_ls[:]:
            print("inner ",file.replace(key, i))
            value_ls.remove(i)
            print(value_ls)
        dictionary_lst.pop(key)
        # Key Error - you already popped the butter when you want to pop it again
        return recur_replace(key, file, dictionary_lst)

def main(dictionary_lst):
    filename = "cookies.txt"

    #print("THIS IS THE DICT TYPE",type(dictionary_lst))
    #sentenc_arr = [] #not sure if you need this
    word_ls = []
    #just reading through the file
    file_token = open(filename, "r")
    for file in file_token:
        body = file.split("-")
        for line in body: #body is all sentences
            sentc = line.split(",")
            #sentc is only a single sentence, last one
            #new_body = "".join(sentc)
            #print(sentc)
            #sentenc_arr.append(sentc)

    #join the sentence so that only one variable is edited
    #print(type(sentc))
    #print("THIS IS THE OLD FILE VARIABLE", file)
    new_file = " ".join(body)
    #new_file = []
    ##print("THIS IS THE JOINED SENTENCES", new_file)
    '''value = dictionary_lst.values()
    print("this are the value of dict ", value)
    key = dictionary_lst.get(key)
    print("this are the key of dict", key) '''
    for key in dictionary_lst.iterkeys():
            recur_replace(key, new_file, vegan_ls)

    print("outer ", new_file)

main(vegan_ls)


# Programa para ler se o a no é BISSEXTO ou não.
# 1 importar a biblioteca datetime, escolher apenas a função date que é a que  será usada
from datetime import date
ano = int(input('Que ano quer analisar? '))
# usar a formula IF(SE) para analisar se o ano digitado for 0, será igual a date.today().year.
# date.today() irá pegar a data de hoje e o .year irá trazer o que eu quero da data de hoje que no caso é o ano
# então se o usuario digitar 0 o programa irá trazer o ano de hoje automaticamente.
if ano == 0:
    ano = date.today().year
# aqui é usado o se(IF) com o and para fazer um outro se(IF) dentro dele. analisar duas situações diferentes.
if ano % 4 == 0 and ano % 100 != 0 or ano % 400 == 0:
    print('O ano {} é BISSEXTO'.format(ano))
else:
    print('O ano {} não é BISSEXTO'.format(ano))


# 拯救007，使用深度优先搜索方法
class StepNode():
    def __init__(self, cur, cho):
        self.current = cur
        self.choose = cho

class Node():
    def __init__(self, index, x, y):
        self.index = index
        self.x = x
        self.y = y
        self.canJumpTo = None

def getInput():
    n, x = 1, 0
    count = 0
    cro = []
    while n != 0:
        s = input().split(" ")
        i, j = [eval(k) for k in s]
        if x == 0:
            n, x = i, j
        else:
            if i**2 + j**2 > 7.5**2 and abs(i) <= 50 and abs(j) <= 50:
                cro.append(Node(count, i, j))
                count += 1
            n -= 1
    return x, cro

def firstJump():
    if jump >= 50 - 7.5:
        return True
    else:
        ch = []
        for i in range(len(crocodiles)):
            if crocodiles[i].x**2 + crocodiles[i].y**2 <= (jump + radius)**2:
                ch.append(i)
        step.append(StepNode(-1, ch))

def dfsJump():
    def isSave(i):
        if 50-abs(i.x) <= jump or 50-abs(i.y) <= jump:
            return True
        else:
            return False

    jumpPath = []
    jumpTo = []
    k = False
    def dfs(i):
        nonlocal k
        if visit[i.index] == 0:
            visit[i.index] = 1
            if isSave(i):
                k = True
            else:
                ch = []
                for j in range(len(crocodiles)):
                    if (crocodiles[j].x-i.x)**2 + (crocodiles[j].y-i.y)**2 <= jump**2:
                        ch.append(j)
                i.canJumpTo = ch

                jumpTo.append(i.index)

                for j in i.canJumpTo:
                    k = dfs(crocodiles[j])
                    if k:
                        m = jumpTo.copy()
                        m.append(j)
                        jumpPath.append(m)
                        k = False
                else:
                    jumpTo.pop()
        return k

    for i in step[0].choose:
        if visit[i] == 0:
            k = dfs(crocodiles[i])
    return jumpPath

jump, crocodiles = getInput()
visit = [0] * len(crocodiles)
step = []
radius = 7.5

if firstJump():
    print(1)
else:
    jumpPath = dfsJump()
    if len(jumpPath) == 0:
        print(0)
    else:
        jumpPath.sort(key=len)
        # print(jumpPath)
        minPath = jumpPath[0]
        print(len(minPath)+1)
        for i in minPath:
            print(crocodiles[i].x, crocodiles[i].y)


from Project.classes.CsvWriter import CSVWriter
from .Parser import Parser
from bricsagentapplication.model.models import Publication, Organization
from .Service import Service
from bricsagentapplication.cache.Cache import Cache
from webdriver_manager.firefox import GeckoDriverManager
from selenium import webdriver
from selenium.webdriver.firefox.options import Options

class Agent:
    def __init__(self):
        self.service = Service()
        self.cache = Cache()
        self.csv_writer = CSVWriter()
        self.parser = Parser()
        self.in_progress = False
        self.links_count = 0
        self.current_link_number = 0
        self.current_country = ""

    def __new__(cls):
        if not hasattr(cls, 'instance'):
            print('creating new agent')
            cls.instance = super(Agent, cls).__new__(cls)
        return cls.instance

    def create_driver(self):
        opts = Options()
        opts.headless = True
        return webdriver.Firefox(options=opts, executable_path=GeckoDriverManager().install())

    def fill_db_for_country(self, country):
        self.in_progress = True
        self.current_link_number = 0
        self.current_country = country

        driver = self.create_driver()
        self.parser.driver = driver

        links, links_journals = self.parser.get_btns_links_and_journals(country)
        self.csv_writer.write_links_journals_functions_files(links, links_journals)

        links_journals = self.csv_writer.read_links_journals_file(country)
        self.links_count = len(links_journals)

        for link_to_button in links_journals:
            self.current_link_number += 1
            print(country + ", link: ", link_to_button)
            try:
                Publication.objects.get(link_to_btn=link_to_button)
            except Publication.DoesNotExist:
                try:
                    journal_name = links_journals[link_to_button]
                    publication_info = self.parser.get_publication_information(link_to_button, journal_name)
                    if publication_info is not None:
                        publication_info.country = country
                        self.service.save_publication_info(publication_info)
                except KeyError:
                    my_file = open("exceptions.txt", "a")
                    my_file.write('\n we do not know journal of ' + link_to_button + " is\n ")
                    my_file.close()
                    continue

        print("Done")
        self.in_progress = False
        driver.quit()
        # для тестирования: выбираем журнал

        # ! springer
        # if 'Nature' in journal_name:

        # ! sciencedirect
        # if 'Earth and Planetary Science Letters' in journal_name or 'Geochimica et Cosmochimica Acta' in journal_name or 'Water Research' in journal_name:

        # ! acs
        # if 'Organic Letters' in journal_name or 'Macromolecules' in journal_name or 'Journal of the American Chemical Society' in journal_name or 'Inorganic Chemistry' in journal_name or 'The Journal of Physical Chemistry Letters' in journal_name or 'Analytical Chemistry' in journal_name or 'Environmental Science and Technology' in journal_name or 'ACS Nano' in journal_name or 'Nano Letters' in journal_name:
        #
        # ! rsc
        # if 'Chemical Communications' in journal_name or 'Chemical Science' in journal_name:
        #
        # ! pnasorg
        # if 'Proceedings of the National Academy of Sciences of the United States of America' in journal_name:
        #
        # ! advances science
        # if 'Science' == journal_name or 'Science Advances' == journal_name or 'Science Translational Medicine' == journal_name:
        #
        # ! journalsplos
        # if 'PLOS Biology' == journal_name or 'PLOS Genetics' == journal_name:
        #
        # ! jneurosci
        # if 'Journal of Neuroscience' == journal_name or 'Cancer Research' == journal_name:
        #
        # ! jbcorg
        # if 'Journal of Biological Chemistry' == journal_name:
        #
        #  ! jciorg
        #  if 'Journal of Clinical Investigation' == journal_name:
        #
        #  ! cell
        #  if 'Cell' == journal_name:
        #
        #  ! royal
        #  if 'Proceedings of the Royal Society B' == journal_name:
        #
        #  ! meta aip scitation
        #  if 'Applied Physics Letters' == journal_name:
        #
        #         # ? academic oup
        #         # if 'Monthly Notices of the Royal Astronomical Society: Letters' == journal_name:
        #
        #         # ? iop science
        #         # if 'The Astrophysical Journal Letters' == journal_name:
        #
        # ! meta ru press
        # if 'Journal of Experimental Medicine' == journal_name or 'Journal of Cell Biology' == journal_name:
        #
        # ! genome
        # if 'Genome Research' == journal_name:
        #
        #         # ? planet cell
        #         # if 'The Plant Cell' == journal_name:
        #
        #         # ? aanda
        #         # if 'Astronomy & Astrophysics' in journal_name:
        #
        # ! pubs geo science world
        #  https: // www.natureindex.com / article / 10.1130 / g47008.1
        #  https: // www.natureindex.com / article / 10.1130 / g46740.1
        #  https: // www.natureindex.com / article / 10.1130 / g46923.1
        #  https: // www.natureindex.com / article / 10.1130 / g46180.1
        #  if 'Geology' in journal_name:
        #
        # ! wiley
        #  if 'Advanced Functional Materials' == journal_name or 'Advanced Materials' == journal_name or 'Geophysical Research Letters' == journal_name or 'Journal of Geophysical Research: Atmospheres' == journal_name or 'Journal of Geophysical Research: Solid Earth' == journal_name or 'Ecology Letters' == journal_name or 'Angewandte Chemie International Edition' == journal_name:
        #
        # ! journals aps
        # if 'Physical Review' in journal_name:
        #
        # ! elife
        # if 'eLife' == journal_name:
        #
        # ! cell ajhg
        #  if 'American Journal of Human Genetics' == journal_name or 'Cancer Cell' == journal_name or 'Neuron' == journal_name or 'Cell Metabolism ' == journal_name or 'Cell Host & Microbe' == journal_name or 'Cell Stem Cell' == journal_name or 'Current Biology' == journal_name or 'Immunity' == journal_name:

        # self.update_cache(self.parser)

    # def get_top_unis_names(self, country):
    #     additional_driver = self.create_driver()
    #     topOrganizationsNames = self.parser.get_top_unis_names(country, additional_driver)
    #     additional_driver.quit()
    #
    #     organizationsTop = []
    #
    #     for org in topOrganizationsNames:
    #         organizationsTop.append({'id': University.objects.get(name=org).id, 'name': org})
    #     return organizationsTop

    def get_filling_progress(self):
        if self.links_count != 0:
            progress = self.current_link_number/self.links_count*100
        else:
            progress = 0
        return {"country": self.current_country, "progress": progress, "in_progress": self.in_progress}

    # def update_cache(self, parser):
        # todo заменить
        # for country in ['Brazil', 'Russia', 'India', 'China', 'South Africa']:
        #     self.cache.cache_countries_orgs_top(parser.get_country_top_unis_names(country), country)


#function to pull data from cvs file
def listCharacters():
    people =csv.DictReader(open("starwars.csv","r"))
    for entry in people:
        print(entry)

#print out character name, actor name, and birthdate

#main
print(listCharacters())

#Part 2:
import datetime
def getBios():
    people=csv.DictReader(open("starwars.csv","r"))
    now=datetime.date.today()
    for entry in people:
        birth_parts=entry["birthday"].split("/")
        print(birth_parts)
        #print(birth_parts)
        birthday=datetime.date(int(birth_parts[2]),int(birth_parts[0]),int(birth_parts[1]))
        age = now - birthday
        print(entry["name"],"is played by",entry["character"],"and was born on",birthday.strftime("%B %d, %Y"),"and is",round(age.days/365),"years old")

#main
getBios()


# -*- coding: utf-8 -*-
import collections
import subprocess

import pytest

def pytest_addoption(parser):
    group = parser.getgroup("circleci-parallelized")
    group.addoption(
        "--circleci-parallelize",
        dest="circleci_parallelize",
        action="store_true",
        default=False,
        help="Enable parallelization across CircleCI containers.",
    )

def get_verbosity(config):
    return config.option.verbose

def circleci_parallelized_enabled(config):
    return config.getoption("circleci_parallelize")

def pytest_report_collectionfinish(config, startdir, items):
    if circleci_parallelized_enabled(config):
        verbosity = get_verbosity(config)
        if verbosity == 0:
            return "running {} items due to CircleCI parallelism".format(len(items))
        elif verbosity > 0:
            return "running {} items due to CircleCI parallelism: {}".format(
                len(items), ", ".join(map(get_class_name, items))
            )
    else:
        return ""

def get_class_name(item):
    class_name, module_name = None, None
    for parent in reversed(item.listchain()):
        if isinstance(parent, pytest.Class):
            class_name = parent.name
        elif isinstance(parent, pytest.Module):
            module_name = parent.module.__name__
            break

    if class_name:
        return "{}.{}".format(module_name, class_name)
    else:
        return module_name

def filter_tests_with_circleci(test_list):
    circleci_input = "\n".join(test_list).encode("utf-8")
    p = subprocess.Popen(
        [
            "circleci",
            "tests",
            "split",
            "--split-by=timings",
            "--timings-type=classname",
        ],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
    )
    circleci_output, _ = p.communicate(circleci_input)
    return [
        line.strip() for line in circleci_output.decode("utf-8").strip().split("\n")
    ]

@pytest.hookimpl(hookwrapper=True)
def pytest_cmdline_main(config):
    outcome = yield config
    exit_code = outcome.get_result()

    # Exit Code 5 indicates no tests were collected
    # https://docs.pytest.org/en/7.1.x/reference/exit-codes.html
    # If there are more workers than tests, this can cause "No Tests" error
    # This is fine, and we cast it to an OK exit code
    if circleci_parallelized_enabled(config) and exit_code == 5:
        outcome.force_result(0)

def pytest_collection_modifyitems(session, config, items):
    if not circleci_parallelized_enabled(config):
        return

    class_mapping = collections.defaultdict(list)
    for item in items:
        class_name = get_class_name(item)
        class_mapping[class_name].append(item)

    filtered_tests = filter_tests_with_circleci(class_mapping.keys())

    new_items = []
    for name in filtered_tests:
        new_items.extend(class_mapping[name])

    items[:] = new_items


# from map2loop import m2l_topology
import networkx as nx
import random
import numpy as np
import pandas as pd
import os

##########################################################################
# Save out and compile taskfile needed to generate geomodeller model using the geomodellerbatch engine
#
# loop2geomodeller(test_data_path,tmp_path,output_path,save_faults,compute_etc)
# Args:
# test_data_path root directory of test data
# tmp_path directory of temporary outputs
# output_path directory of outputs
# ave_faults flag for saving faults or not
# compute_etc flag for actual calculations or just project output
#
# Creates geomodeller taskfile files from varous map2loop outputs
##########################################################################
def loop2geomodeller(model_name,test_data_path,tmp_path,output_path,dtm_file,bbox,model_top,model_base,save_faults,compute_etc,workflow):

    f=open(test_data_path+'/'+model_name+'/m2l.taskfile','w')
    f.write('#---------------------------------------------------------------\n')
    f.write('#-----------------------Project Header-----------------------\n')
    f.write('#---------------------------------------------------------------\n')
    f.write('name: "UWA_Intrepid"\n')
    f.write('description: "Automate_batch_Model"\n')
    f.write('    GeomodellerTask {\n')
    f.write('    CreateProject {\n')
    f.write('        name: "Hamersley"\n')
    f.write('        author: "Mark"\n')
    f.write('        date: "23/10/2019  0: 0: 0"\n')
    f.write('        projection { map_projection: "GDA94 / MGA50"}\n')
    f.write('        version: "2.0"\n')
    f.write('        units: meters\n')
    f.write('        precision: 1.0\n')
    f.write('        Extents {\n')
    f.write('            xmin: '+str(bbox[0])+'\n')
    f.write('            ymin: '+str(bbox[1])+'\n')
    f.write('            zmin: '+str(model_base)+'\n')
    f.write('            xmax: '+str(bbox[2])+'\n')
    f.write('            ymax: '+str(bbox[3])+'\n')
    f.write('            zmax: '+str(model_top)+'\n')
    f.write('        }\n')
    f.write('        deflection2d: 0.001\n')
    f.write('        deflection3d: 0.001\n')
    f.write('        discretisation: 10.0\n')
    f.write('        referenceTop: false\n')
    f.write('        CustomDTM {\n')
    f.write('            Extents {\n')
    f.write('            xmin: '+str(bbox[0])+'\n')
    f.write('            ymin: '+str(bbox[1])+'\n')
    f.write('            xmax: '+str(bbox[2])+'\n')
    f.write('            ymax: '+str(bbox[3])+'\n')
    f.write('            }\n')
    f.write('            name: "Topography"\n')
    f.write('            filename {\n')
    f.write('                Grid_Name: "'+dtm_file+'"\n')
    f.write('            }\n')
    f.write('            nx: 10\n')
    f.write('            ny: 10\n')
    f.write('        }\n')
    f.write('    }\n')
    f.write('}\n')

    orientations=pd.read_csv(output_path+'orientations_clean.csv',',')
    contacts=pd.read_csv(output_path+'contacts_clean.csv',',')
    all_sorts=pd.read_csv(tmp_path+'all_sorts_clean.csv',',')

    empty_fm=[]

    for indx,afm in all_sorts.iterrows():
        foundcontact=False
        for indx2,acontact in contacts.iterrows():
            if(acontact['formation'] in afm['code']):
                foundcontact=True
                break
        foundorientation=False
        for indx3,ano in orientations.iterrows():
            if(ano['formation'] in afm['code']):
                foundorientation=True
                break
        if(not foundcontact or not foundorientation):
            empty_fm.append(afm['code'])

    #print(empty_fm)

    all_sorts=np.genfromtxt(tmp_path+'all_sorts_clean.csv',delimiter=',',dtype='U100')
    nformations=len(all_sorts)

    f.write('#---------------------------------------------------------------\n')
    f.write('#-----------------------Create Formations-----------------------\n')
    f.write('#---------------------------------------------------------------\n')

    for i in range (1,nformations):
        if( not all_sorts[i,4] in empty_fm):
            f.write('GeomodellerTask {\n')
            f.write('CreateFormation {\n')

            ostr='    name: "'+all_sorts[i,4].replace("\n","")+'"\n'
            f.write(ostr)

            ostr='    red: '+str(random.randint(1,256)-1)+'\n'
            f.write(ostr)

            ostr='    green: '+str(random.randint(1,256)-1)+'\n'
            f.write(ostr)

            ostr='    blue: '+str(random.randint(1,256)-1)+'\n'
            f.write(ostr)

            f.write('    }\n')
            f.write('}\n')

    f.write('#---------------------------------------------------------------\n')
    f.write('#-----------------------Set Stratigraphic Pile------------------\n')
    f.write('#---------------------------------------------------------------\n')

    for i in range (1,nformations):
    #for i in range (nformations-1,0,-1):
        if(all_sorts[i,2]==str(1)):
            f.write('GeomodellerTask {\n')
            f.write('SetSeries {\n')

            ostr='    name: "'+all_sorts[i][5].replace("\n","")+'"\n'
            f.write(ostr)

            ostr='    position: 1\n'
            f.write(ostr)

            ostr='    relation: "erode"\n'
            f.write(ostr)

            f.write('    }\n')
            f.write('}\n')

            for j in range(nformations-1,0,-1):
    #        for j in range(1,nformations):
                if(all_sorts[j,1]==all_sorts[i,1]):
                    if( not all_sorts[j][4] in empty_fm):
                        f.write('GeomodellerTask {\n')
                        f.write('AddFormationToSeries {\n')

                        ostr='    series: "'+all_sorts[j][5]+'"\n'
                        f.write(ostr)

                        ostr='    formation: "'+all_sorts[j][4]+'"\n'
                        f.write(ostr)

                        f.write('    }\n')
                        f.write('}\n')

    if(save_faults):
        output_path=test_data_path+'output/'

        faults_len=pd.read_csv(output_path+'fault_dimensions.csv')

        n_allfaults=len(faults_len)

        fcount=0
        for i in range(0,n_allfaults):
            f.write('GeomodellerTask {\n')
            f.write('CreateFault {\n')
            ostr='    name: "'+faults_len.iloc[i]["Fault"]+'"\n'
            f.write(ostr)

            ostr='    red: '+str(random.randint(1,256)-1)+'\n'
            f.write(ostr)

            ostr='    green: '+str(random.randint(1,256)-1)+'\n'
            f.write(ostr)

            ostr='    blue: '+str(random.randint(1,256)-1)+'\n'
            f.write(ostr)

            f.write('    }\n')
            f.write('}\n')
            fcount=fcount+1

            f.write('GeomodellerTask {\n')
            f.write('    Set3dFaultLimits {\n')
            f.write('        Fault_name: "'+faults_len.iloc[i]["Fault"]+ '"\n')
            f.write('        Horizontal: '+str(faults_len.iloc[i]["HorizontalRadius"])+ '\n')
            f.write('        Vertical: '+str(faults_len.iloc[i]["VerticalRadius"])+ '\n')
            f.write('        InfluenceDistance: '+str(faults_len.iloc[i]["InfluenceDistance"])+ '\n')
            f.write('    }\n')
            f.write('}\n')

    f.write('#---------------------------------------------------------------\n')
    f.write('#-----------------------Import 3D contact data ---Base Model----\n')
    f.write('#---------------------------------------------------------------\n')

    contacts=pd.read_csv(output_path+'contacts_clean.csv',',')
    all_sorts=pd.read_csv(tmp_path+'all_sorts_clean.csv',',')
    #all_sorts.set_index('code',  inplace = True)
    #display(all_sorts)

    for inx,afm in all_sorts.iterrows():
        #print(afm[0])
        if( not afm['code'] in empty_fm):
            f.write('GeomodellerTask {\n')
            f.write('    Add3DInterfacesToFormation {\n')
            f.write('          formation: "'+str(afm['code'])+'"\n')

            for indx2,acontact in contacts.iterrows():
                if(acontact['formation'] in afm['code'] ):
                    ostr='              point {x:'+str(acontact['X'])+'; y:'+str(acontact['Y'])+'; z:'+str(acontact['Z'])+'}\n'
                    f.write(ostr)
            f.write('    }\n')
            f.write('}\n')
    f.write('#---------------------------------------------------------------\n')
    f.write('#------------------Import 3D orientation data ---Base Model-----\n')
    f.write('#---------------------------------------------------------------\n')

    orientations=pd.read_csv(output_path+'orientations_clean.csv',',')
    all_sorts=pd.read_csv(tmp_path+'all_sorts_clean.csv',',')
    #all_sorts.set_index('code',  inplace = True)
    #display(all_sorts)

    for inx,afm in all_sorts.iterrows():
        #print(groups[agp])
        if( not afm['code'] in empty_fm):
            f.write('GeomodellerTask {\n')
            f.write('    Add3DFoliationToFormation {\n')
            f.write('          formation: "'+str(afm['code'])+'"\n')
            for indx2,ano in orientations.iterrows():
                if(ano['formation'] in afm['code']):
                    f.write('           foliation {\n')
                    ostr='                  Point3D {x:'+str(ano['X'])+'; y:'+str(ano['Y'])+'; z:'+str(ano['Z'])+'}\n'
                    f.write(ostr)
                    ostr='                  direction: '+str(ano['azimuth'])+'\n'
                    f.write(ostr)
                    ostr='                  dip: '+str(ano['dip'])+'\n'
                    f.write(ostr)
                    if(ano['polarity']==1):
                        ostr='                  polarity: Normal_Polarity\n'
                    else:
                        ostr='                  polarity: Reverse_Polarity\n'
                    f.write(ostr)
                    ostr='           }\n'
                    f.write(ostr)
            f.write('    }\n')
            f.write('}\n')

    f.write('#---------------------------------------------------------------\n')
    f.write('#-----------------------Import 3D fault data ---Base Model------\n')
    f.write('#---------------------------------------------------------------\n')

    contacts=pd.read_csv(output_path+'faults.csv',',')
    faults=pd.read_csv(output_path+'fault_dimensions.csv',',')

    for indx,afault in faults.iterrows():
        f.write('GeomodellerTask {\n')
        f.write('    Add3DInterfacesToFormation {\n')
        f.write('          formation: "'+str(afault['Fault'])+'"\n')
        for indx2,acontact in contacts.iterrows():
            if(acontact['formation'] == afault['Fault']):
                ostr='              point {x:'+str(acontact['X'])+'; y:'+str(acontact['Y'])+'; z:'+str(acontact['Z'])+'}\n'
                f.write(ostr)
        f.write('    }\n')
        f.write('}\n')

    f.write('#---------------------------------------------------------------\n')
    f.write('#------------------Import 3D fault orientation data ------------\n')
    f.write('#---------------------------------------------------------------\n')

    orientations=pd.read_csv(output_path+'fault_orientations.csv',',')
    faults=pd.read_csv(output_path+'fault_dimensions.csv',',')

    for indx,afault in faults.iterrows():
        f.write('GeomodellerTask {\n')
        f.write('    Add3DFoliationToFormation {\n')
        f.write('          formation: "'+str(afault['Fault'])+'"\n')
        for indx2,ano in orientations.iterrows():
            if(ano['formation'] == afault['Fault']):
                f.write('           foliation {\n')
                ostr='                  Point3D {x:'+str(ano['X'])+'; y:'+str(ano['Y'])+'; z:'+str(ano['Z'])+'}\n'
                f.write(ostr)
                ostr='                  direction: '+str(ano['DipDirection'])+'\n'
                f.write(ostr)
                if(ano['dip'] == -999):
                    ostr='                  dip: '+str(random.randint(60,90))+'\n'
                else:
                    ostr='                  dip: '+str(ano['dip'])+'\n'
                f.write(ostr)
                if(ano['DipPolarity']==1):
                    ostr='                  polarity: Normal_Polarity\n'
                else:
                    ostr='                  polarity: Reverse_Polarity\n'
                f.write(ostr)
                ostr='           }\n'
                f.write(ostr)
        f.write('    }\n')
        f.write('}\n')

    if(save_faults):
        G=nx.read_gml(tmp_path+"fault_network.gml",label='label')
        #nx.draw(G, with_labels=True, font_weight='bold')
        edges=list(G.edges)
        #for i in range(0,len(edges)):
            #print(edges[i][0],edges[i][1])
        cycles=list(nx.simple_cycles(G))
        #display(cycles)
        f.write('#---------------------------------------------------------------\n')
        f.write('#-----------------------Link faults with faults ----------------\n')
        f.write('#---------------------------------------------------------------\n')
        f.write('GeomodellerTask {\n')
        f.write('    LinkFaultsWithFaults {\n')

        for i in range(0,len(edges)):
                found=False
                for j in range(0,len(cycles)):
                    if(edges[i][0]== cycles[j][0] and edges[i][1]== cycles[j][1]):
                        found=True # fault pair is first two elements in a cycle list so don't save to taskfile
                if(not found):
                    ostr='        FaultStopsOnFaults{ fault: "'+edges[i][1]+'"; stopson: "'+edges[i][0]+'"}\n'
                    f.write(ostr)

        f.write('    }\n')
        f.write('}\n')

    if(save_faults):
        all_fault_group=np.genfromtxt(output_path+'group-fault-relationships.csv',delimiter=',',dtype='U100')
        ngroups=len(all_fault_group)
        all_fault_group=np.transpose(all_fault_group)
        nfaults=len(all_fault_group)

        f.write('#---------------------------------------------------------------\n')
        f.write('#-----------------------Link series with faults ----------------\n')
        f.write('#---------------------------------------------------------------\n')
        f.write('GeomodellerTask {\n')
        f.write('    LinkFaultsWithSeries {\n')

        for i in range(1,nfaults):
            first=True
            for j in range(1,ngroups):
                if(all_fault_group[i,j]==str(1)):
                    if(first):
                        ostr='    FaultSeriesLinks{ fault: "'+all_fault_group[i,0]+'"; series: ['
                        f.write(ostr)
                        ostr='"'+all_fault_group[0,j]+'"'
                        f.write(ostr)
                        first=False
                    else:
                        ostr=', "'+all_fault_group[0,j]+'"'
                        f.write(ostr)
            if(not first):
                ostr=']}\n'
                f.write(ostr)

        f.write('    }\n')
        f.write('}\n')

    f.write('GeomodellerTask {\n')
    f.write('    SaveProjectAs {\n')
    f.write('        filename: "./'+model_name+'.xml"\n')
    f.write('    }\n')
    f.write('}\n')
    f.close()

    if(compute_etc):
        f=open(test_data_path+model_name+'/'+'m2l_compute.taskfile','w')
        f.write('#---------------------------------------------------------------\n')
        f.write('#----------------------------Load Model----------------------\n')
        f.write('#---------------------------------------------------------------\n')
        f.write('GeomodellerTask {\n')
        f.write('    OpenProjectNoGUI {\n')
        f.write('        filename: "./'+model_name+'.xml"\n')
        f.write('    }\n')
        f.write('}\n')

        f.write('#---------------------------------------------------------------\n')
        f.write('#----------------------------Compute Model----------------------\n')
        f.write('#---------------------------------------------------------------\n')
        f.write('\n')
        f.write('GeomodellerTask {\n')
        f.write('    ComputeModel {\n')
        f.write('        SeriesList {\n')
        f.write('            node: "All" \n')
        f.write('        }\n')
        f.write('        SectionList {\n')
        f.write('            node: "All"\n')
        f.write('        }\n')
        f.write('        FaultList {\n')
        f.write('            node: "All"\n')
        f.write('        }\n')
        f.write('        radius: 10.0\n')
        f.write('    }\n')
        f.write('}\n')

        f.write('#---------------------------------------------------------------\n')
        f.write('#-----------------------Add geophysical Properties--------------\n')
        f.write('#---------------------------------------------------------------\n')
        f.write('\n')
        f.write('\n')
        f.write('\n')
        f.write('#---------------------------------------------------------------\n')
        f.write('#--------------------------Export Lithology Voxet---------------\n')
        f.write('#---------------------------------------------------------------\n')
        f.write('GeomodellerTask {\n')
        f.write('    SaveLithologyVoxet {\n')
        f.write('        nx: 25\n')
        f.write('        ny: 25\n')
        f.write('        nz: 40\n')
        f.write('        LithologyVoxetFileStub: "./Litho_Voxet/LithoVoxet.vo"\n')
        f.write('    }\n')
        f.write('}\n')
        f.write('#---------------------------------------------------------------\n')
        f.write('#--------------------------Save As Model------------------------\n')
        f.write('#---------------------------------------------------------------\n')
        f.write('\n')

        f.write('GeomodellerTask {\n')
        f.write('    SaveProjectAs {\n')
        f.write('        filename: "/'+model_name+'.xml"\n')
        f.write('    }\n')
        f.write('}\n')
        f.write('GeomodellerTask {\n')
        f.write('    CloseProjectNoGUI {\n')
        f.write('    }\n')
        f.write('}\n')

        f.close()

# same same expect it builds a list that then gets written all at once (this version is slower!)
def loop2geomodeller2(model_name,test_data_path,tmp_path,output_path,dtm_file,bbox,save_faults,compute_etc,workflow):

    f=open(test_data_path+'/'+model_name+'/m2l.taskfile','w')
    ostr=[]

    ostr.append('#---------------------------------------------------------------\n')
    ostr.append('#-----------------------Project Header-----------------------\n')
    ostr.append('#---------------------------------------------------------------\n')
    ostr.append('name: "UWA_Intrepid"\n')
    ostr.append('description: "Automate_batch_Model"\n')
    ostr.append('    GeomodellerTask {\n')
    ostr.append('    CreateProject {\n')
    ostr.append('        name: "Hamersley"\n')
    ostr.append('        author: "Mark"\n')
    ostr.append('        date: "23/10/2019  0: 0: 0"\n')
    ostr.append('        projection { map_projection: "GDA94 / MGA50"}\n')
    ostr.append('        version: "2.0"\n')
    ostr.append('        units: meters\n')
    ostr.append('        precision: 1.0\n')
    ostr.append('        Extents {\n')
    ostr.append('            xmin: '+str(bbox[0])+'\n')
    ostr.append('            ymin: '+str(bbox[1])+'\n')
    ostr.append('            zmin: -7000\n')
    ostr.append('            xmax: '+str(bbox[2])+'\n')
    ostr.append('            ymax: '+str(bbox[3])+'\n')
    ostr.append('            zmax: 1200\n')
    ostr.append('        }\n')
    ostr.append('        deflection2d: 0.001\n')
    ostr.append('        deflection3d: 0.001\n')
    ostr.append('        discretisation: 10.0\n')
    ostr.append('        referenceTop: false\n')
    ostr.append('        CustomDTM {\n')
    ostr.append('            Extents {\n')
    ostr.append('            xmin: '+str(bbox[0])+'\n')
    ostr.append('            ymin: '+str(bbox[1])+'\n')
    ostr.append('            xmax: '+str(bbox[2])+'\n')
    ostr.append('            ymax: '+str(bbox[3])+'\n')
    ostr.append('            }\n')
    ostr.append('            name: "Topography"\n')
    ostr.append('            filename {\n')
    ostr.append('                Grid_Name: "'+dtm_file+'"\n')
    ostr.append('            }\n')
    ostr.append('            nx: 10\n')
    ostr.append('            ny: 10\n')
    ostr.append('        }\n')
    ostr.append('    }\n')
    ostr.append('}\n')

    orientations=pd.read_csv(output_path+'orientations_clean.csv',',')
    contacts=pd.read_csv(output_path+'contacts_clean.csv',',')
    all_sorts=pd.read_csv(tmp_path+'all_sorts_clean.csv',',')

    empty_fm=[]

    for indx,afm in all_sorts.iterrows():
        foundcontact=False
        for indx2,acontact in contacts.iterrows():
            if(acontact['formation'] in afm['code']):
                foundcontact=True
                break
        foundorientation=False
        for indx3,ano in orientations.iterrows():
            if(ano['formation'] in afm['code']):
                foundorientation=True
                break
        if(not foundcontact or not foundorientation):
            empty_fm.append(afm['code'])

    #print(empty_fm)

    all_sorts=np.genfromtxt(tmp_path+'all_sorts_clean.csv',delimiter=',',dtype='U100')
    nformations=len(all_sorts)

    ostr.append('#---------------------------------------------------------------\n')
    ostr.append('#-----------------------Create Formations-----------------------\n')
    ostr.append('#---------------------------------------------------------------\n')

    for i in range (1,nformations):
        if( not all_sorts[i,4] in empty_fm):
            ostr.append('GeomodellerTask {\n')
            ostr.append('CreateFormation {\n')

            ostr2='    name: "'+all_sorts[i,4].replace("\n","")+'"\n'
            ostr.append(ostr2)

            ostr2='    red: '+str(random.randint(1,256)-1)+'\n'
            ostr.append(ostr2)

            ostr2='    green: '+str(random.randint(1,256)-1)+'\n'
            ostr.append(ostr2)

            ostr2='    blue: '+str(random.randint(1,256)-1)+'\n'
            ostr.append(ostr2)

            ostr.append('    }\n')
            ostr.append('}\n')

    ostr.append('#---------------------------------------------------------------\n')
    ostr.append('#-----------------------Set Stratigraphic Pile------------------\n')
    ostr.append('#---------------------------------------------------------------\n')

    for i in range (1,nformations):
    #for i in range (nformations-1,0,-1):
        if(all_sorts[i,2]==str(1)):
            ostr.append('GeomodellerTask {\n')
            ostr.append('SetSeries {\n')

            ostr2='    name: "'+all_sorts[i][5].replace("\n","")+'"\n'
            ostr.append(ostr2)

            ostr2='    position: 1\n'
            ostr.append(ostr2)

            ostr2='    relation: "erode"\n'
            ostr.append(ostr2)

            ostr.append('    }\n')
            ostr.append('}\n')

            for j in range(nformations-1,0,-1):
    #        for j in range(1,nformations):
                if(all_sorts[j,1]==all_sorts[i,1]):
                    if( not all_sorts[j][4] in empty_fm):
                        ostr.append('GeomodellerTask {\n')
                        ostr.append('AddFormationToSeries {\n')

                        ostr2='    series: "'+all_sorts[j][5]+'"\n'
                        ostr.append(ostr2)

                        ostr2='    formation: "'+all_sorts[j][4]+'"\n'
                        ostr.append(ostr2)

                        ostr.append('    }\n')
                        ostr.append('}\n')

    if(save_faults):
        output_path=test_data_path+'output/'

        faults_len=pd.read_csv(output_path+'fault_dimensions.csv')

        n_allfaults=len(faults_len)

        fcount=0
        for i in range(0,n_allfaults):
            ostr.append('GeomodellerTask {\n')
            ostr.append('CreateFault {\n')
            ostr2='    name: "'+faults_len.iloc[i]["Fault"]+'"\n'
            ostr.append(ostr2)

            ostr2='    red: '+str(random.randint(1,256)-1)+'\n'
            ostr.append(ostr2)

            ostr2='    green: '+str(random.randint(1,256)-1)+'\n'
            ostr.append(ostr2)

            ostr2='    blue: '+str(random.randint(1,256)-1)+'\n'
            ostr.append(ostr2)

            ostr.append('    }\n')
            ostr.append('}\n')
            fcount=fcount+1

            ostr.append('GeomodellerTask {\n')
            ostr.append('    Set3dFaultLimits {\n')
            ostr.append('        Fault_name: "'+faults_len.iloc[i]["Fault"]+ '"\n')
            ostr.append('        Horizontal: '+str(faults_len.iloc[i]["HorizontalRadius"])+ '\n')
            ostr.append('        Vertical: '+str(faults_len.iloc[i]["VerticalRadius"])+ '\n')
            ostr.append('        InfluenceDistance: '+str(faults_len.iloc[i]["InfluenceDistance"])+ '\n')
            ostr.append('    }\n')
            ostr.append('}\n')

    ostr.append('#---------------------------------------------------------------\n')
    ostr.append('#-----------------------Import 3D contact data ---Base Model----\n')
    ostr.append('#---------------------------------------------------------------\n')

    contacts=pd.read_csv(output_path+'contacts_clean.csv',',')
    all_sorts=pd.read_csv(tmp_path+'all_sorts_clean.csv',',')
    #all_sorts.set_index('code',  inplace = True)
    #display(all_sorts)

    for inx,afm in all_sorts.iterrows():
        #print(afm[0])
        if( not afm['code'] in empty_fm):
            ostr.append('GeomodellerTask {\n')
            ostr.append('    Add3DInterfacesToFormation {\n')
            ostr.append('          formation: "'+str(afm['code'])+'"\n')

            for indx2,acontact in contacts.iterrows():
                if(acontact['formation'] in afm['code'] ):
                    ostr2='              point {x:'+str(acontact['X'])+'; y:'+str(acontact['Y'])+'; z:'+str(acontact['Z'])+'}\n'
                    ostr.append(ostr2)
            ostr.append('    }\n')
            ostr.append('}\n')
    ostr.append('#---------------------------------------------------------------\n')
    ostr.append('#------------------Import 3D orientation data ---Base Model-----\n')
    ostr.append('#---------------------------------------------------------------\n')

    orientations=pd.read_csv(output_path+'orientations_clean.csv',',')
    all_sorts=pd.read_csv(tmp_path+'all_sorts_clean.csv',',')
    #all_sorts.set_index('code',  inplace = True)
    #display(all_sorts)

    for inx,afm in all_sorts.iterrows():
        #print(groups[agp])
        if( not afm['code'] in empty_fm):
            ostr.append('GeomodellerTask {\n')
            ostr.append('    Add3DFoliationToFormation {\n')
            ostr.append('          formation: "'+str(afm['code'])+'"\n')
            for indx2,ano in orientations.iterrows():
                if(ano['formation'] in afm['code']):
                    ostr.append('           foliation {\n')
                    ostr2='                  Point3D {x:'+str(ano['X'])+'; y:'+str(ano['Y'])+'; z:'+str(ano['Z'])+'}\n'
                    ostr.append(ostr2)
                    ostr2='                  direction: '+str(ano['azimuth'])+'\n'
                    ostr.append(ostr2)
                    ostr2='                  dip: '+str(ano['dip'])+'\n'
                    ostr.append(ostr2)
                    if(ano['polarity']==1):
                        ostr2='                  polarity: Normal_Polarity\n'
                    else:
                        ostr2='                  polarity: Reverse_Polarity\n'
                    ostr.append(ostr2)
                    ostr2='           }\n'
                    ostr.append(ostr2)
            ostr.append('    }\n')
            ostr.append('}\n')

    ostr.append('#---------------------------------------------------------------\n')
    ostr.append('#-----------------------Import 3D fault data ---Base Model------\n')
    ostr.append('#---------------------------------------------------------------\n')

    contacts=pd.read_csv(output_path+'faults.csv',',')
    faults=pd.read_csv(output_path+'fault_dimensions.csv',',')

    for indx,afault in faults.iterrows():
        ostr.append('GeomodellerTask {\n')
        ostr.append('    Add3DInterfacesToFormation {\n')
        ostr.append('          formation: "'+str(afault['Fault'])+'"\n')
        for indx2,acontact in contacts.iterrows():
            if(acontact['formation'] == afault['Fault']):
                ostr2='              point {x:'+str(acontact['X'])+'; y:'+str(acontact['Y'])+'; z:'+str(acontact['Z'])+'}\n'
                ostr.append(ostr2)
        ostr.append('    }\n')
        ostr.append('}\n')

    ostr.append('#---------------------------------------------------------------\n')
    ostr.append('#------------------Import 3D fault orientation data ------------\n')
    ostr.append('#---------------------------------------------------------------\n')

    orientations=pd.read_csv(output_path+'fault_orientations.csv',',')
    faults=pd.read_csv(output_path+'fault_dimensions.csv',',')

    for indx,afault in faults.iterrows():
        ostr.append('GeomodellerTask {\n')
        ostr.append('    Add3DFoliationToFormation {\n')
        ostr.append('          formation: "'+str(afault['Fault'])+'"\n')
        for indx2,ano in orientations.iterrows():
            if(ano['formation'] == afault['Fault']):
                ostr.append('           foliation {\n')
                ostr2='                  Point3D {x:'+str(ano['X'])+'; y:'+str(ano['Y'])+'; z:'+str(ano['Z'])+'}\n'
                ostr.append(ostr2)
                ostr2='                  direction: '+str(ano['DipDirection'])+'\n'
                ostr.append(ostr2)
                if(ano['dip'] == -999):
                    ostr2='                  dip: '+str(random.randint(60,90))+'\n'
                else:
                    ostr2='                  dip: '+str(ano['dip'])+'\n'
                ostr.append(ostr2)
                if(ano['DipPolarity']==1):
                    ostr2='                  polarity: Normal_Polarity\n'
                else:
                    ostr2='                  polarity: Reverse_Polarity\n'
                ostr.append(ostr2)
                ostr2='           }\n'
                ostr.append(ostr2)
        ostr.append('    }\n')
        ostr.append('}\n')

    if(save_faults):
        G=nx.read_gml(tmp_path+"fault_network.gml",label='label')
        #nx.draw(G, with_labels=True, font_weight='bold')
        edges=list(G.edges)
        #for i in range(0,len(edges)):
            #print(edges[i][0],edges[i][1])
        cycles=list(nx.simple_cycles(G))
        #display(cycles)
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('#-----------------------Link faults with faults ----------------\n')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('GeomodellerTask {\n')
        ostr.append('    LinkFaultsWithFaults {\n')

        for i in range(0,len(edges)):
                found=False
                for j in range(0,len(cycles)):
                    if(edges[i][0]== cycles[j][0] and edges[i][1]== cycles[j][1]):
                        found=True # fault pair is first two elements in a cycle list so don't save to taskfile
                if(not found):
                    ostr2='        FaultStopsOnFaults{ fault: "'+edges[i][1]+'"; stopson: "'+edges[i][0]+'"}\n'
                    ostr.append(ostr2)

        ostr.append('    }\n')
        ostr.append('}\n')

    if(save_faults):
        all_fault_group=np.genfromtxt(output_path+'group-fault-relationships.csv',delimiter=',',dtype='U100')
        ngroups=len(all_fault_group)
        all_fault_group=np.transpose(all_fault_group)
        nfaults=len(all_fault_group)

        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('#-----------------------Link series with faults ----------------\n')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('GeomodellerTask {\n')
        ostr.append('    LinkFaultsWithSeries {\n')

        for i in range(1,nfaults):
            first=True
            for j in range(1,ngroups):
                if(all_fault_group[i,j]==str(1)):
                    if(first):
                        ostr2='    FaultSeriesLinks{ fault: "'+all_fault_group[i,0]+'"; series: ['
                        ostr.append(ostr2)
                        ostr2='"'+all_fault_group[0,j]+'"'
                        ostr.append(ostr2)
                        first=False
                    else:
                        ostr2=', "'+all_fault_group[0,j]+'"'
                        ostr.append(ostr2)
            if(not first):
                ostr2=']}\n'
                ostr.append(ostr2)

        ostr.append('    }\n')
        ostr.append('}\n')

    ostr.append('GeomodellerTask {\n')
    ostr.append('    SaveProjectAs {\n')
    ostr.append('        filename: "./'+model_name+'.xml"\n')
    ostr.append('    }\n')
    ostr.append('}\n')
    f.close()

    if(compute_etc):
        f=open(test_data_path+model_name+'/'+'m2l_compute.taskfile','w')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('#----------------------------Load Model----------------------\n')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('GeomodellerTask {\n')
        ostr.append('    OpenProjectNoGUI {\n')
        ostr.append('        filename: "./'+model_name+'.xml"\n')
        ostr.append('    }\n')
        ostr.append('}\n')

        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('#----------------------------Compute Model----------------------\n')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('\n')
        ostr.append('GeomodellerTask {\n')
        ostr.append('    ComputeModel {\n')
        ostr.append('        SeriesList {\n')
        ostr.append('            node: "All" \n')
        ostr.append('        }\n')
        ostr.append('        SectionList {\n')
        ostr.append('            node: "All"\n')
        ostr.append('        }\n')
        ostr.append('        FaultList {\n')
        ostr.append('            node: "All"\n')
        ostr.append('        }\n')
        ostr.append('        radius: 10.0\n')
        ostr.append('    }\n')
        ostr.append('}\n')

        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('#-----------------------Add geophysical Properties--------------\n')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('\n')
        ostr.append('\n')
        ostr.append('\n')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('#--------------------------Export Lithology Voxet---------------\n')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('GeomodellerTask {\n')
        ostr.append('    SaveLithologyVoxet {\n')
        ostr.append('        nx: 25\n')
        ostr.append('        ny: 25\n')
        ostr.append('        nz: 40\n')
        ostr.append('        LithologyVoxetFileStub: "./Litho_Voxet/LithoVoxet.vo"\n')
        ostr.append('    }\n')
        ostr.append('}\n')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('#--------------------------Save As Model------------------------\n')
        ostr.append('#---------------------------------------------------------------\n')
        ostr.append('\n')

        ostr.append('GeomodellerTask {\n')
        ostr.append('    SaveProjectAs {\n')
        ostr.append('        filename: "/'+model_name+'.xml"\n')
        ostr.append('    }\n')
        ostr.append('}\n')
        ostr.append('GeomodellerTask {\n')
        ostr.append('    CloseProjectNoGUI {\n')
        ostr.append('    }\n')
        ostr.append('}\n')
        f.writelines(ostr)
        f.close()

from pyamg import solve
def solve_pyamg(A,B):
    return solve(A,B,verb=False,tol=1e-8)

##########################################################################
# Import outputs from map2loop to LoopStructural and view with Lavavu
#
# loop2LoopStructural(thickness_file,orientation_file,contacts_file,bbox)
# Args:
# thickness_file path of fornation thickness file
# orientation_file path of orientations file
# contacts_file path of contacts file
# bbox model bounding box
#
# Calculates model and displays in LavaVu wthin notebook
##########################################################################
def loop2LoopStructural(thickness_file,orientation_file,contacts_file,bbox):
    from LoopStructural import GeologicalModel
    from LoopStructural.visualisation import LavaVuModelViewer
    import lavavu

    df = pd.read_csv(thickness_file)

    thickness = {}
    for f in df['formation'].unique():
        thickness[f] = np.mean(df[df['formation']==f]['thickness'])

    #display(thickness)
    order = ['P__TKa_xs_k','P__TKo_stq','P__TKk_sf','P__TK_s',
    'A_HAu_xsl_ci', 'A_HAd_kd', 'A_HAm_cib', 'A_FOj_xs_b',
    'A_FO_xo_a', 'A_FO_od', 'A_FOu_bbo',
    'A_FOp_bs', 'A_FOo_bbo', 'A_FOh_xs_f', 'A_FOr_b']

    strat_val = {}
    val = 0
    for o in order:
        if o in thickness:
            strat_val[o] = val
            val+=thickness[o]

    #display(strat_val)

    orientations = pd.read_csv(orientation_file)
    contacts = pd.read_csv(contacts_file)

    contacts['val'] = np.nan

    for o in strat_val:
        contacts.loc[contacts['formation']==o,'val'] = strat_val[o]
    data = pd.concat([orientations,contacts],sort=False)
    data['type'] = np.nan
    for o in order:
        data.loc[data['formation']==o,'type'] = 's0'
    data

    boundary_points = np.zeros((2,3))
    boundary_points[0,0] = bbox[0]
    boundary_points[0,1] = bbox[1]
    boundary_points[0,2] = -20000
    boundary_points[1,0] = bbox[2]
    boundary_points[1,1] = bbox[3]
    boundary_points[1,2] = 1200

    model = GeologicalModel(boundary_points[0,:],boundary_points[1,:])
    model.set_model_data(data)
    strati = model.create_and_add_foliation('s0', #identifier in data frame
                                                        interpolatortype="FDI", #which interpolator to use
                                                        nelements=400000, # how many tetras/voxels
                                                        buffer=0.1, # how much to extend nterpolation around box
                                                        solver='external',
                                                        external=solve_pyamg
                                                       )
    #viewer = LavaVuModelViewer()
    viewer = LavaVuModelViewer(model)
    viewer.add_data(strati['feature'])
    viewer.add_isosurface(strati['feature'],
    #                       nslices=10,
                          slices= strat_val.values(),
    #                     voxet={'bounding_box':boundary_points,'nsteps':(100,100,50)},
                          paint_with=strati['feature'],
                          cmap='tab20'

                         )
    #viewer.add_scalar_field(model.bounding_box,(100,100,100),
   #                           'scalar',
    ##                             norm=True,
    #                         paint_with=strati['feature'],
    #                         cmap='tab20')
    viewer.add_scalar_field(strati['feature'])
    viewer.set_viewer_rotation([-53.8190803527832, -17.1993350982666, -2.1576387882232666])
    #viewer.save("fdi_surfaces.png")
    viewer.interactive()

##########################################################################
# Import outputs from map2loop to gempy and view with pyvtk
# loop2gempy(test_data_name,tmp_path,vtk_pth,orientations_file,contacts_file,groups_file,dtm_reproj_file,bbox,model_base, model_top,vtk)
# Args:
# test_data_name root name of project
# tmp_path path of temp files directory
# vtk_pth path of vtk output directory
# orientations_file path of orientations file
# contacts_file path of contacts file
# groups_file path of groups file
# dtm_reproj_file path of dtm file
# bbox model bounding box
# model_base z value ofbase of model
# model_top z value of top of model
# vtk flag as to wether to save out model to vtk
#
# Calculates model and displays in external vtk viewer
##########################################################################
def loop2gempy(test_data_name: str, tmp_path: str, vtk_path: str, orientations_file: str,
               contacts_file: str, groups_file:str,
               bbox: tuple, model_base: float, model_top: float, vtk: bool, dtm_reproj_file:str = None,
               va=None,
               verbose: bool = False, compute: bool = True):
    """

    :param test_data_name:
    :param tmp_path:
    :param vtk_path:
    :param orientations_file:
    :param contacts_file:
    :param groups_file:
    :param bbox:
    :param model_base:
    :param model_top:
    :param vtk:
    :param dtm_reproj_file:
    :param va: vertical anisotropy. Factor by which all Z coordinates are multiplied by
    :param verbose:
    :param compute:
    :return:
    """
    import gempy as gp
    from gempy import plot

    geo_model = gp.create_model(test_data_name)

    # If depth coordinates are much smaller than XY the whole system of equations becomes very unstable. Until
    # I fix it properly in gempy this is a handcrafted hack
    if va is None:
        va = (float(bbox[0]) - float(bbox[2])) / (model_base - model_top)/2

        if va < 3:
            va = 0
        else:
            print('The vertical exageration is: ', va)

    gp.init_data(geo_model, extent=[bbox[0], bbox[2], bbox[1], bbox[3], model_base * va, model_top * va],
                 resolution=(50, 50, 50),
                 path_o=orientations_file,
                 path_i=contacts_file)

    geo_model.modify_surface_points(geo_model.surface_points.df.index, Z=geo_model.surface_points.df['Z'] * va)

    if dtm_reproj_file is not None:
        # Load reprojected topography to model

        fp = dtm_reproj_file
        geo_model.set_topography(source='gdal', filepath=fp)

        # Rescaling topography:
        geo_model.grid.topography.values[:, 2] *= va
        geo_model.grid.update_grid_values()
        geo_model.update_from_grid()

    # Pile processing:
    contents = np.genfromtxt(groups_file,
                             delimiter=',', dtype='U100')

    # Init dictionary Series:Surfaces
    map_series_to_surfaces = {}
    choice = 0
    for group in contents:
        # Reading surfaces groups
        surfaces_g = np.atleast_2d(np.genfromtxt(tmp_path + group + '.csv', delimiter=',', dtype='U100'))

        # Check if there are several choices
        if surfaces_g.shape[1] > 1:
            surfaces_g = surfaces_g[choice]
        # Deleting the first element since it is not a surface
        surfaces_g = surfaces_g[1:]
        # Creating the mapping dictionary
        map_series_to_surfaces[group] = surfaces_g.tolist()

    if verbose is True:
        print(map_series_to_surfaces)

    # Setting pile to model
    gp.map_series_to_surfaces(geo_model, map_series_to_surfaces, remove_unused_series=False)

    #    Removing related data
    del_surfaces = geo_model.surfaces.df.groupby('series').get_group('Default series')['surface']
    geo_model.delete_surfaces(del_surfaces, remove_data=True)

    # Removing series that have not been mapped to any surface
    geo_model.delete_series('Default series')

    if compute is True:
        gp.set_interpolator(geo_model, theano_optimizer='fast_run', dtype='float64')
        gp.compute_model(geo_model)

    # Visualise Model
    gp.plot.plot_3D(geo_model, render_data=False)

    # Save model as vtk
    if vtk:
        gp.plot.export_to_vtk(geo_model, path=vtk_path, name=test_data_name + '.vtk', voxels=False, block=None,
                              surfaces=True)

    return geo_model

def loop2gempy_(test_data_name, tmp_path, vtk_path, orientations_file, contacts_file, groups_file, dtm_reproj_file,
                bbox, model_base, model_top, vtk):
    import gempy as gp
    from gempy import plot
    geo_model = gp.create_model(test_data_name)

    # If depth coordinates are much smaller than XY the whole system of equations becomes very unstable. Until
    # I fix it properly in gempy this is a handcrafted hack
    ve = (bbox[0] - bbox[2]) / (model_base - model_top)

    if ve < 3:
        ve = 0
    else:
        print('The vertical exageration is: ', ve)

    gp.init_data(geo_model, extent=[bbox[0], bbox[2], bbox[1], bbox[3], model_base*ve, model_top*ve],
        resolution = (50,50,50),
          path_o = orientations_file,
          path_i = contacts_file, default_values=True);

    # Show example lithological points
    #gp.get_data(geo_model, 'surface_points').head()

    # Show example orientations
    #gp.get_data(geo_model, 'orientations').head()

    # Plot some of this data
    #gp.plot.plot_data(geo_model, direction='z')

    geo_model.modify_surface_points(geo_model.surface_points.df.index, Z=geo_model.surface_points.df['Z']*ve)

    # Load reprojected topgraphy to model

    fp = dtm_reproj_file
    geo_model.set_topography(source='gdal',filepath=fp)

    contents=np.genfromtxt(groups_file,delimiter=',',dtype='U100')
    ngroups=len(contents)

    faults = gp.Faults()
    series = gp.Series(faults)
    #series.df

    #display(ngroups,contents)
    groups = []

    for i in range (0,ngroups):
        groups.append(contents[i].replace("\n",""))
        series.add_series(contents[i].replace("\n",""))
        print(contents[i].replace("\n",""))

    series.delete_series('Default series')

    #series

    # Load surfaces and assign to series
    surfaces = gp.Surfaces(series)

    print(ngroups,groups)
    for i in range(0,ngroups):
        contents=np.genfromtxt(tmp_path+groups[i]+'.csv',delimiter=',',dtype='U100')
        nformations=len(contents.shape)

        if(nformations==1):
            for j in range (1,len(contents)):
                surfaces.add_surface(str(contents[j]).replace("\n",""))
                d={groups[i]:str(contents[j]).replace("\n","")}
                surfaces.map_series({groups[i]:(str(contents[j]).replace("\n",""))}) #working but no gps
        else:
            for j in range (1,len(contents[0])):
                surfaces.add_surface(str(contents[0][j]).replace("\n",""))
                d={groups[i]:str(contents[0][j]).replace("\n","")}
                surfaces.map_series({groups[i]:(str(contents[0][j]).replace("\n",""))}) #working but no gps

    # Set Interpolation Data
    id_only_one_bool = geo_model.surface_points.df['id'].value_counts() == 1
    id_only_one = id_only_one_bool.index[id_only_one_bool]
    single_vals = geo_model.surface_points.df[geo_model.surface_points.df['id'].isin(id_only_one)]
    for idx, vals in single_vals.iterrows():
        geo_model.add_surface_points(vals['X'], vals['Y'], vals['Z'], vals['surface'])

    geo_model.update_structure()

    gp.set_interpolation_data(geo_model,
                              compile_theano=True,
                              theano_optimizer='fast_compile',
                              verbose=[])

    # Provide summary data on model

    #geo_model.additional_data.structure_data

    #Calculate Model
    gp.compute_model(geo_model)

    # Extract surfaces to visualize in 3D renderers
    #gp.plot.plot_section(geo_model, 49, direction='z', show_data=False)

    ver, sim = gp.get_surfaces(geo_model)

    # import winsound
    # duration = 700  # milliseconds
    # freq = 1100  # Hz
    # winsound.Beep(freq, duration)
    # winsound.Beep(freq, duration)
    # winsound.Beep(freq, duration)

    #Visualise Model
    gp.plot.plot_3D(geo_model, render_data=False)

    #Save model as vtk
    if(vtk):
        gp.plot.export_to_vtk(geo_model, path=vtk_path, name=test_data_name+'.vtk', voxels=False, block=None, surfaces=True)

    return geo_model