YEDDA.py

'''
 # @ Author: Jie Yang
 # @ Create Time: 2018
 # @ Last Modified by: Jie Yang  Contact: jieynlp@gmail.com
 # @ Last Modified time: 2022-04-20 12:38:31
 '''
# -*- coding: utf-8 -*-
import os.path
import platform
from tkinter import filedialog
from tkinter import font
from tkinter import messagebox
from collections import deque
from tkinter import *
from tkinter.ttk import Frame, Button, Radiobutton, Label, Combobox
from tkinter.simpledialog import Dialog
from tkinter.scrolledtext import ScrolledText
from dataclasses import dataclass
from typing import List, Optional, Tuple
import json

from utils.recommend import *


class Editor(ScrolledText):
    def __init__(self, parent, entity_pattern, recommend_pattern):
        super().__init__(parent, selectbackground='light salmon')
        self.entity_pattern = entity_pattern
        self.recommend_pattern = recommend_pattern
        fnt = font.Font(family='Times', size=20, weight="bold", underline=0)
        self.config(insertbackground='red', insertwidth=4, font=fnt)

        def _ignore(_): return 'break'

        # Disable the default copy behaviour when right click.
        # For MacOS, right click is button 2, other systems are button3
        self.bind('<Button-2>', _ignore)
        self.bind('<Button-3>', _ignore)
        self.set_colors(None)

    def set_colors(self, colors: Optional[List[Tuple[str, str]]]):
        """
        Set colors for different entity type
        :param colors: list of (entity, color), or None to disable colorful annotation
        """
        self.colors = colors
        for t in self.tag_names():
            if t.startswith('entity') or t.startswith('recommend'):
                self.tag_delete(t)
        # TODO color edge to discriminate recommend
        self.tag_configure("edge", background="light grey", foreground='DimGrey', font=('Times', 12))
        if colors is None:
            self.tag_configure("recommend", background='light green')
            self.tag_configure("entity", background="SkyBlue1")
        else:
            for label, color in self.colors:
                self.tag_configure('entity_' + label, background=color)
                self.tag_configure('recommend_' + label, background=color)

    def _highlight_entity(self, start: str, count: int, tag_name: str):
        end = f'{start}+{count}c'
        sharp_pos = self.get(start, end).rfind('#')
        word_start = f"{start}+2c"
        word_end = f"{start}+{sharp_pos}c"
        if self.colors:
            label_start = f'{start}+{sharp_pos + 1}c'
            label_end = f'{start}+{count - 2}c'
            label = self.get(label_start, label_end)
            tag_name = f'{tag_name}_{label}'
        self.tag_add(tag_name, word_start, word_end)
        self.tag_add("edge", start, word_start)
        self.tag_add("edge", word_end, end)

    def show_annotation_tag(self, show: bool):
        self.tag_configure('edge', elide=not show)

    def highlight_recommend(self, start: str, count: int):
        self._highlight_entity(start, count, 'recommend')

    def highlight_entity(self, start: str, count: int):
        self._highlight_entity(start, count, 'entity')

    def get_text(self) -> str:
        """get text from 0 to end"""
        return self.get("1.0", "end-1c")

    def _highlight_entities(self, pattern, highlight_func):
        count_var = StringVar()
        from_index = '1.0'
        while True:
            pos = self.search(pattern, from_index, END, count=count_var, regexp=True)
            if pos == "":
                break
            from_index = f"{pos}+{count_var.get()}c"
            highlight_func(pos, int(count_var.get()))

    def update_view(self):
        self._highlight_entities(self.entity_pattern, self.highlight_entity)
        self._highlight_entities(self.recommend_pattern, self.highlight_recommend)

    def current_entity(self) -> (str, (str, int)):
        def find_pattern_span_in_line(pattern):
            row, col = self.index(INSERT).split('.')
            cursor_col = int(col)
            count_var = StringVar()
            from_index = f'{row}.0'
            while True:
                pos = self.search(pattern, from_index, f'{row}.end', count=count_var, regexp=True)
                if pos == '':
                    break
                row, col = pos.split('.')
                match_end = f'{row}.{int(col) + int(count_var.get())}'  # here don't use offset form
                if int(col) < cursor_col < (int(col) + int(count_var.get())):
                    return pos, match_end
                from_index = match_end
            return None

        span = find_pattern_span_in_line(self.entity_pattern)
        if span is not None:
            return 'gold', span
        span = find_pattern_span_in_line(self.recommend_pattern)
        if span is not None:
            return 'recommend', span
        else:
            return None, (None, None)

    def get_selection(self) -> Optional[str]:
        try:
            return self.selection_get()
        except TclError:
            return None


@dataclass
class KeyDef:
    key: str
    name: str
    desc: str = ''
    color: str = None


class KeyMapFrame(Frame):
    def __init__(self, parent, keymap: List[KeyDef]):
        super().__init__(parent, relief='groove')
        self.keymap = sorted(keymap, key=lambda x: x.key)
        self.rows = len(keymap)
        self.textFontStyle = 'Times'
        self.key_labels = []
        self.name_entries = []
        self.create_widgets()

    def create_widgets(self):
        title = Label(self, text="Shortcuts map", foreground="blue", font=(self.textFontStyle, 14, "bold"))
        title.grid(row=0, column=0, columnspan=2, sticky=W, padx=6, pady=8)
        for row, item in enumerate(self.keymap, 1):
            key_lbl = Label(self, text=item.key.upper() + ": ", font=(self.textFontStyle, 14, "bold"))
            key_lbl.grid(row=row, column=0, sticky=NW, padx=4, pady=4)
            self.key_labels.append(key_lbl)

            name_entry = Entry(self, font=(self.textFontStyle, 14), bg=item.color)
            name_entry.insert(0, item.name)
            name_entry.grid(row=row, column=1, columnspan=1, rowspan=1, sticky=NW, padx=4, pady=4)
            self.name_entries.append(name_entry)

    def update_keymap(self, keymap):
        self.keymap = sorted(keymap, key=lambda x: x.key)
        for lbl in self.key_labels:
            lbl.destroy()
        for ent in self.name_entries:
            ent.destroy()
        self.key_labels = []
        self.name_entries = []
        self.create_widgets()

    def read_keymap(self) -> List[KeyDef]:
        """read current keymap in GUI, might be changed by user"""
        new_map = []
        for i, cmd in enumerate(self.keymap):
            new_name = self.name_entries[i].get()
            if new_name.strip() != '':
                new_map.append(KeyDef(cmd.key, new_name, cmd.desc, cmd.color))
            else:
                print(f'{cmd.key} key deleted')
        return new_map


class QueryExport(Dialog):
    def __init__(self, parent, filename, sample):
        self.confirmed = False
        self.sample = sample
        super().__init__(parent, 'Exporting ' + filename)  # here dialog shows

    def body(self, master):
        """override"""
        box = Frame(master, relief='groove')
        self.scheme_var = StringVar(master, "BMES")
        Radiobutton(box, text="BMES", variable=self.scheme_var, value="BMES").pack(side=LEFT, padx=5, pady=5)
        Radiobutton(box, text="BIO", variable=self.scheme_var, value="BIO").pack(side=LEFT, padx=5, pady=5)
        box.pack()
        self.segmented_var = BooleanVar(master, self._guess_segmented())
        Checkbutton(master, text="Segmented", variable=self.segmented_var).pack()
        self.only_NP_var = BooleanVar(master, False)
        Checkbutton(master, text="Only NP label", variable=self.only_NP_var).pack()
        self.export_recommended_var = BooleanVar(master, True)
        Checkbutton(master, text="Export Recommended", variable=self.export_recommended_var).pack()

    def apply(self):
        """override, called after press ok, not called on cancel"""
        self.confirmed = True

    def segmented(self) -> bool:
        return self.segmented_var.get()

    def only_NP(self) -> bool:
        return self.only_NP_var.get()

    def keep_recommended(self) -> bool:
        return self.export_recommended_var.get()

    def tag_scheme(self) -> str:
        return self.scheme_var.get()

    def _guess_segmented(self):
        """False for non-segmented Chinese, True for English or Segmented Chinese.
        Make naive guess, user should check whether the guess is right
        """
        ascii_percent = sum(1 for c in self.sample if c.isascii()) / len(self.sample)
        is_english = (ascii_percent > 0.8)
        space_percent = self.sample.count(' ') / len(self.sample)
        many_space = (space_percent > 0.2)
        return is_english or many_space or False


def all_colors():
    colors = []
    for color in ('LightBlue', 'LightCyan', 'LightGoldenrod', 'LightPink',
                  'LightSalmon', 'LightSkyBlue', 'LightSteelBlue', 'LightYellow'):
        colors += [c + n for c, n in zip([color] * 5, ['', '1', '2', '3', '4'])]
    return sorted(colors, key=lambda c: list(reversed(c)))


class Application(Frame):
    def __init__(self, parent):
        super().__init__(parent)
        self.Version = "YEDDA-V1.0 Annotator"
        self.configFile = "configs/default.config"
        self.OS = platform.system().lower()
        self.fileName = ""
        self.file_encoding = 'utf-8'
        self.debug = False
        self.history = deque(maxlen=20)

        # default GUI display parameter
        self.readConfig()
        self.textColumn = 5


        self.entity_regex = r'\[\@.*?\#.*?\*\](?!\#)'
        self.recommendRe = r'\[\$.*?\#.*?\*\](?!\#)'
        self.goldAndrecomRe = r'\[[\@\$)].*?\#.*?\*\](?!\#)'
        self.textFontStyle = "Times"
        self.initUI()

    def KeyDef2Dic(self):
        config_dic = {}
        for item in self.pressCommand:
            config_dic[item.key] = item.name
        return config_dic

    def readConfig(self):
        self.pressCommand = []
        with open(self.configFile, 'r') as fp:
            config_dict = json.load(fp)
        for index,entity in config_dict.items():
            self.pressCommand.append(KeyDef(index,entity))
        for key, color in zip(self.pressCommand, all_colors()):
            key.color = color

        # default GUI display parameter
        self.textRow = max(len(self.pressCommand), 20)


    def initUI(self):
        self.master.title(self.Version)
        self.pack(fill=BOTH, expand=True)

        for i in range(0, self.textColumn):
            self.columnconfigure(i, weight=2)
        # self.columnconfigure(0, weight=2)
        
        self.columnconfigure(self.textColumn + 2, weight=1)
        self.columnconfigure(self.textColumn + 4, weight=1)
        for i in range(0, 16):
            self.rowconfigure(i, weight=1)

        self.filename_lbl = Label(self, text="File: no file is opened")
        self.filename_lbl.grid(sticky=W, pady=4, padx=5)
        self.text = Editor(self, self.entity_regex, self.recommendRe)
        self.text.grid(row=1, column=0, columnspan=self.textColumn, rowspan=self.textRow, padx=12, sticky=NSEW)

        btn = Button(self, text="Open", command=self.onOpen)
        btn.grid(row=1, column=self.textColumn + 1)
        btn = Button(self, text="ReMap", command=self.renewPressCommand)
        btn.grid(row=2, column=self.textColumn + 1, pady=4)
        btn = Button(self, text="NewMap", command=self.savenewPressCommand)
        btn.grid(row=3, column=self.textColumn + 1, pady=4)
        btn = Button(self, text="Export", command=self.generateSequenceFile)
        btn.grid(row=4, column=self.textColumn + 1, pady=4)

        self.use_recommend = BooleanVar(self, True)
        check = Checkbutton(self, text='Recommend', command=self.toggle_use_recommend, variable=self.use_recommend)
        check.grid(row=5, column=self.textColumn + 1, sticky=W, pady=4)

        show_tags_var = BooleanVar(self, True)
        check = Checkbutton(self, text='Show Tags', variable=show_tags_var,
                            command=lambda: self.text.show_annotation_tag(show_tags_var.get()))
        check.grid(row=6, column=self.textColumn + 1, sticky=W)

        self.use_colorful_var = BooleanVar(self, False)
        check = Checkbutton(self, text='Colorful', variable=self.use_colorful_var, command=self.toggle_use_colorful)
        check.grid(row=7, column=self.textColumn + 1, sticky=W)

        self.cursor_index_label = Label(self, text="Ln 1, Col 0")
        self.cursor_index_label.grid(row=self.textRow + 1, sticky=NSEW, pady=4, padx=4)
        cmd_var = StringVar()
        cmd_var.trace_add('write', lambda _, _1, _2: self.preview_cmd_range())
        self.entry = Entry(self, validate='focus', vcmd=self.preview_cmd_range, textvariable=cmd_var)
        self.entry.grid(row=self.textRow + 1, column=1, columnspan=self.textColumn - 2, sticky=NSEW, pady=4, padx=8)
        self.entry.bind('<FocusOut>', self.clear_preview_mark)
        self.entry.bind('<Return>', self.execute_command)

        btn = Button(self, text="Enter", command=lambda: self.execute_command(None))
        btn.grid(row=self.textRow + 1, column=self.textColumn - 1)

        all_keys = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
        for press_key in all_keys:
            self.text.bind(press_key, self.alphanum_key_pressed, add='')
            if self.OS != "windows":
                self.text.bind("<Control-Key-" + press_key + ">", self.keepCurrent)
                self.text.bind("<Control-Key-" + press_key + ">", self.keepCurrent)

        self.text.bind('<Control-Key-z>', self.backToHistory)

        self.text.bind('<Double-Button-1>', self.doubleLeftClick)
        self.text.bind('<ButtonRelease-1>', self.show_cursor_pos)
        self.text.bind('<KeyRelease>', self.show_cursor_pos)

        self.keymap_frame = KeyMapFrame(self, self.pressCommand)
        self.keymap_frame.grid(row=1, column=self.textColumn + 2, rowspan=self.keymap_frame.rows,
                               columnspan=2, padx=6, pady=6, sticky=NW)

        Label(self, text="KeyMap Templates:").grid(row=8, column=self.textColumn + 1)
        self.configListBox = Combobox(self, values=getConfigList(), state='readonly')
        self.configListBox.grid(row=8, column=self.textColumn + 2, columnspan=2)
        # select current config file
        self.configListBox.set(self.configFile.split(os.sep)[-1])
        self.configListBox.bind('<<ComboboxSelected>>', self.on_select_configfile)

    def show_cursor_pos(self, _):
        cursor_index = self.text.index(INSERT)
        row, col = cursor_index.split('.')
        self.cursor_index_label.config(text=f"Ln {row}, Col {col}")

    # TODO: select entity by double left click
    def doubleLeftClick(self, _):
        if self.debug:
            print("Action Track: doubleLeftClick")
        pass
        # cursor_index = self.text.index(INSERT)
        # start_index = ("%s - %sc" % (cursor_index, 5))
        # end_index = ("%s + %sc" % (cursor_index, 5))
        # self.text.tag_add('SEL', '1.0',"end-1c")

    def toggle_use_recommend(self):
        if not self.use_recommend.get():
            content = self.text.get_text()
            content = removeRecommendContent(content, self.recommendRe)
            self.writeFile(self.fileName, content, '1.0')

    def toggle_use_colorful(self):
        if self.use_colorful_var.get():
            self.text.set_colors([(d.name, d.color) for d in self.pressCommand])
        else:
            self.text.set_colors(None)
        self.text.update_view()

    def onOpen(self):
        filename = filedialog.askopenfilename(
            filetypes=[('all files', '.*'), ('text files', '.txt'), ('ann files', '.ann')])
        if filename != '':
            self.text.delete("1.0", END)
            text = self.readFile(filename)
            self.text.insert(END, text)
            self.filename_lbl.config(text="File: " + filename)
            self.autoLoadNewFile(self.fileName, "1.0")
            self.text.mark_set(INSERT, "1.0")
            self.show_cursor_pos(None)

    def readFile(self, filename):
        f = open(filename)
        try:
            text = f.read()
            self.file_encoding = f.encoding
        except UnicodeDecodeError:
            f = open(filename, encoding='utf-8')
            text = f.read()
        self.fileName = filename
        return text

    def setFont(self, value):
        _family = self.textFontStyle
        _size = value
        _weight = "bold"
        _underline = 0
        fnt = font.Font(family=_family, size=_size, weight=_weight, underline=_underline)
        Text(self, font=fnt)

    def setCursorLabel(self, cursor_index):
        row, col = cursor_index.split('.')
        self.cursor_index_label.config(text=f"Ln {row}, Col {col}")

    def clear_preview_mark(self, _):
        self.text.tag_delete('cmd-preview')

    def preview_cmd_range(self):
        preview_tag = 'cmd-preview'
        cmd = self.entry.get().strip()
        self.text.tag_delete(preview_tag)
        self.text.tag_configure(preview_tag, background='light salmon')
        match = re.match(r'^(-?[0-9]+).*', cmd)
        if match:
            count = int(match.group(1))
        else:
            count = 1
        if count > 0:
            self.text.tag_add(preview_tag, INSERT, f'{INSERT}+{count}c')
        else:
            self.text.tag_add(preview_tag, f'{INSERT}-{abs(count)}c', INSERT)
        return True

    def execute_command(self, _):
        self.pushToHistory()
        content = self.entry.get()
        self.clearCommand()
        self.execute_entry_command(content.strip())
        return content

    def alphanum_key_pressed(self, event):
        press_key = event.char
        self.pushToHistory()
        self.clearCommand()
        self.execute_cursor_command(press_key.lower())
        return 'break'

    def backToHistory(self, _):
        if self.debug:
            print("Action Track: backToHistory")
        if len(self.history) > 0:
            content, cursor = self.history.pop()
            self.writeFile(self.fileName, content, cursor)
        else:
            print("History is empty!")

    def keepCurrent(self, _):
        if self.debug:
            print("Action Track: keepCurrent")
        print("keep current, insert:", INSERT)
        print("before:", self.text.index(INSERT))
        self.text.insert(INSERT, 'p')
        print("after:", self.text.index(INSERT))

    def clearCommand(self):
        if self.debug:
            print("Action Track: clearCommand")
        self.entry.delete(0, 'end')

    def execute_cursor_command(self, command):
        print("Command:" + command)
        found, (start, end) = self.text.current_entity()
        selected = self.text.get_selection()
        if not found and not selected:
            print(f'{command} outside entity, no selection, do nothing')
            return
        # selected whole entity, cursor just outside it
        selected_whole = selected is not None and \
                         (re.match(self.entity_regex, selected) or re.match(self.recommendRe, selected))

        # cursor outside existing entity & has selection
        if not found and selected is not None and not selected_whole:
            if self.get_cmd_by_key(command) is None:
                print(f'{command} key not bound, outside entity, do nothing')
                return
            cursor_index = self.text.index(SEL_LAST)
            entity_content, cursor_index = self.replaceString(selected, selected, command, cursor_index)
            above_half = self.text.get('1.0', SEL_FIRST) + entity_content
            below_half = self.text.get(SEL_LAST, "end-1c")
            content = self.addRecommendContent(above_half, below_half, self.use_recommend.get())
            self.writeFile(self.fileName, content, cursor_index)
        # Cursor inside existing entity, no matter has or not has selection.
        # Or Cursor outside existing entity (just on the edge), with the whole entity selected
        else:
            if selected_whole:
                start, end = self.text.index(SEL_FIRST), self.text.index(SEL_LAST)
            covered_string = self.text.get(start, end)
            old_entity, old_label = covered_string.strip('[@$*]').rsplit('#', 1)

            if command == "q":
                print('q: remove entity label')
                new_cursor = f'{end}-{5 + len(old_label)}c'
                entity_content = old_entity
            elif command == 'y':
                print("y: confirm recommend label")
                entity_content = f'[@{old_entity}#{old_label}*]'
                new_cursor = end
            elif len(old_entity) > 0 and self.get_cmd_by_key(command) is not None:
                print(f'{command}: change entity type')
                cmd = self.get_cmd_by_key(command)
                entity_content = f'[@{old_entity}#{cmd.name}*]'
                delta = len(cmd.name) - len(old_label)
                new_cursor = end + (f'+{delta}c' if delta >= 0 else f'{delta}c')
            else:
                print(f'{command}: key not bound, do nothing')
                return
            above_half = self.text.get('1.0', start) + entity_content
            below_half = self.text.get(end, 'end-1c')
            content = self.addRecommendContent(above_half, below_half, self.use_recommend.get())
            self.writeFile(self.fileName, content, new_cursor)

    def execute_entry_command(self, command):
        print(f"EntryCommand: {command}")
        if command == '':  # move to next line
            row, _ = self.text.index(INSERT).split('.')
            self.text.mark_set(INSERT, f'{int(row) + 1}.0')
            self.show_cursor_pos(None)
        elif command.isdigit():
            self.text.mark_set(INSERT, f'{INSERT}+{command}c')
            self.show_cursor_pos(None)
            self.preview_cmd_range()
        elif len(command) >= 2 and command[0] == '-' and command[1:].isdigit():
            self.text.mark_set(INSERT, f'{INSERT}{command}c')
            self.show_cursor_pos(None)
            self.preview_cmd_range()
        else:
            def split_commands(string):
                commands = []
                num = ''
                for c in string:
                    if c.isdigit():
                        num += c
                    else:
                        commands.append((int(num), c))
                        num = ''
                return commands

            for select_num, cmd in split_commands(command):
                assert select_num > 0
                sel_start = self.text.index(INSERT)
                sel_end = self.text.index(f'{INSERT}+{select_num}c')
                selected = self.text.get(sel_start, sel_end)
                if self.get_cmd_by_key(cmd) is not None:
                    above_half = self.text.get('1.0', sel_start)
                    below_half = self.text.get(sel_start, "end-1c")
                    below_half, new_cursor = self.replaceString(below_half, selected, cmd, sel_end)
                    content = self.addRecommendContent(above_half, below_half, self.use_recommend.get())
                    self.writeFile(self.fileName, content, new_cursor)


    def replaceString(self, content, string, replaceType, cursor_index):
        keydef = self.get_cmd_by_key(replaceType)
        if keydef is not None:
            new_string = "[@" + string + "#" + keydef.name + "*]"
            row, col = cursor_index.split('.')
            newcursor_index = f"{row}.{int(col) + len(keydef.name) + 5}"
            content = content.replace(string, new_string, 1)
            return content, newcursor_index
        else:
            print("Invalid command!")
            print("cursor index: ", self.text.index(INSERT))
            return content, cursor_index

    def writeFile(self, fileName, content, newcursor_index):
        print("writeFile")
        if len(fileName) > 0:
            if ".ann" in fileName:
                new_name = fileName
                ann_file = open(new_name, 'w', encoding=self.file_encoding)
                ann_file.write(content)
                ann_file.close()
            else:
                new_name = fileName + '.ann'
                ann_file = open(new_name, 'w', encoding=self.file_encoding)
                ann_file.write(content)
                ann_file.close()
            self.autoLoadNewFile(new_name, newcursor_index)
        else:
            print("Don't write to empty file!")


    def addRecommendContent(self, train_data, decode_data, recommendMode):
        if not recommendMode:
            content = train_data + decode_data
        else:
            if self.debug:
                print("Action Track: addRecommendContent, start Recommend entity")
            content = maximum_matching(train_data, decode_data)
        return content


    def autoLoadNewFile(self, fileName, newcursor_index):
        if self.debug:
            print("Action Track: autoLoadNewFile")
        if len(fileName) > 0:
            self.text.delete("1.0", END)
            text = self.readFile(fileName)
            self.text.insert("end-1c", text)
            self.filename_lbl.config(text="File: " + fileName)
            self.text.mark_set(INSERT, newcursor_index)
            self.text.see(newcursor_index)
            self.show_cursor_pos(None)
            self.text.update_view()


    def pushToHistory(self):
        self.history.append((self.text.get_text(), self.text.index(INSERT)))

    # update shortcut map, directly in current configfile
    def renewPressCommand(self):
        if self.debug:
            print("Action Track: renewPressCommand")
        self.pressCommand = self.keymap_frame.read_keymap()
        with open(self.configFile, 'wb') as fp:
            json.dump(self.KeyDef2Dic(self.pressCommand), fp)
        self.keymap_frame.update_keymap(self.pressCommand)
        messagebox.showinfo("Remap Notification",
                            "Shortcut map has been updated!\n\n" +
                            "Configure file has been saved in File:" + self.configFile)

    # save as new shortcut map
    def savenewPressCommand(self):
        if self.debug:
            print("Action Track: savenewPressCommand")
        self.pressCommand = self.keymap_frame.read_keymap()
        # prompt to ask configFile name
        self.configFile = filedialog.asksaveasfilename(
            initialdir="./configs/",
            title="Save New Config",
            filetypes=(("YEDDA configs", "*.config"), ("all files", "*.*")))
        # change to relative path following self.init()
        self.configFile = os.path.relpath(self.configFile)
        # make sure ending with ".config"
        if not self.configFile.endswith(".config"):
            self.configFile += ".config"
        with open(self.configFile, 'wb') as fp:
            json.dump(self.KeyDef2Dic(self.pressCommand), fp)
        self.keymap_frame.update_keymap(self.pressCommand)
        messagebox.showinfo("Save New Map Notification",
                            "Shortcut map has been saved and updated!\n\n"
                            + "Configure file has been saved in File:" + self.configFile)

    def on_select_configfile(self, event=None):
        if event and self.debug:
            print("Change shortcut map to: ", event.widget.get())
        self.configFile = os.path.join("configs", event.widget.get())
        self.configListBox.set(self.configFile.split(os.sep)[-1])
        self.readConfig()
        self.keymap_frame.update_keymap(self.pressCommand)


    def generateSequenceFile(self):
        if (".ann" not in self.fileName) and (".txt" not in self.fileName):
            out_error = "Export only works on filename ended in .ann or .txt!\nPlease rename file."
            print(out_error)
            messagebox.showerror("Export error!", out_error)
            return -1
        dlg = QueryExport(self, self.fileName, self.text.get_text()[:100])
        if not dlg.confirmed:
            print("Operation canceled")
            return
        fileLines = open(self.fileName, 'r', encoding=self.file_encoding).readlines()
        lineNum = len(fileLines)
        new_filename = self.fileName.split('.ann')[0] + '.' + dlg.tag_scheme().lower()
        seqFile = open(new_filename, 'w', encoding=self.file_encoding)
        for line in fileLines:
            if len(line) <= 2:
                seqFile.write('\n')
                continue
            else:
                if not dlg.keep_recommended():
                    line = removeRecommendContent(line, self.recommendRe)
                    pattern = self.entity_regex
                else:
                    pattern = self.goldAndrecomRe
                wordTagPairs = getWordTagPairs(line, dlg.segmented(), dlg.tag_scheme(), dlg.only_NP(), pattern)
                for wordTag in wordTagPairs:
                    seqFile.write(wordTag)
                # use null line to separate sentences
                seqFile.write('\n')
        seqFile.close()
        print("Exported file into sequence style in file: ", new_filename)
        print("Line number:", lineNum)
        showMessage = "Exported file successfully!\n\n"
        showMessage += "Tag scheme: " + dlg.tag_scheme() + "\n\n"
        showMessage += "Keep Recom: " + str(dlg.keep_recommended()) + "\n\n"
        showMessage += "Text Segmented: " + str(dlg.segmented()) + "\n\n"
        showMessage += "Line Number: " + str(lineNum) + "\n\n"
        showMessage += "Saved to File: " + new_filename
        messagebox.showinfo("Export Message", showMessage)

    def get_cmd_by_key(self, key):
        return next((item for item in self.pressCommand if item.key == key), None)

    def get_cmd_by_name(self, name):
        return next((item for item in self.pressCommand if item.name == name), None)


def getConfigList():
    fileNames = os.listdir("./configs")
    filteredFileNames = sorted(filter(lambda x: (not x.startswith(".")) and (x.endswith(".config")), fileNames))
    return list(filteredFileNames)


def getWordTagPairs(tagedSentence, segmented=True, tagScheme="BMES", onlyNP=False, entityRe=r'\[\@.*?\#.*?\*\]'):
    sentence = tagedSentence.strip('\n')
    tagged_chunks = []
    for match in re.finditer(entityRe, sentence):
        chunk = (match.group(), match.start(), match.end(), True)  # (chunk_of_words, start, end, is_tagged)
        tagged_chunks.append(chunk)

    if len(tagged_chunks) == 0:
        tagged_chunks = [(sentence, 0, len(sentence), False)]  # TODO semantically wrong

    chunks = []
    for idx in range(0, len(tagged_chunks)):
        if idx == 0:
            if tagged_chunks[idx][1] > 0:  # first character is not tagged
                chunks.append((sentence[0:tagged_chunks[idx][1]], 0, tagged_chunks[idx][1], False))
                chunks.append(tagged_chunks[idx])
            else:
                chunks.append(tagged_chunks[idx])
        else:
            if tagged_chunks[idx][1] == tagged_chunks[idx - 1][2]:
                chunks.append(tagged_chunks[idx])
            elif tagged_chunks[idx][1] < tagged_chunks[idx - 1][2]:
                print("ERROR: found pattern has overlap!", tagged_chunks[idx][1], ' with ', tagged_chunks[idx - 1][2])
            else:
                chunks.append(
                    (sentence[tagged_chunks[idx - 1][2]:tagged_chunks[idx][1]], tagged_chunks[idx - 1][2],
                     tagged_chunks[idx][1],
                     False))
                chunks.append(tagged_chunks[idx])

        sent_len = len(sentence)
        if idx == len(tagged_chunks) - 1:
            if tagged_chunks[idx][2] > sent_len:
                print("ERROR: found pattern position larger than sentence length!")
            elif tagged_chunks[idx][2] < sent_len:
                chunks.append([sentence[tagged_chunks[idx][2]:sent_len], tagged_chunks[idx][2], sent_len, False])
            else:
                continue
    return turnFullListToOutputPair(chunks, segmented, tagScheme, onlyNP)


def turnFullListToOutputPair(fullList, segmented=True, tagScheme="BMES", onlyNP=False):
    pair_list = []
    for chunk_words, start, end, is_tagged in fullList:
        if is_tagged:
            plain_words, label = chunk_words.strip('[@$]').rsplit('#', 1)
            label = label.strip('*')
            if segmented:
                plain_words = plain_words.split()
            if onlyNP:
                label = "NP"
            outList = outputWithTagScheme(plain_words, label, tagScheme)
            pair_list.extend(outList)
        else:
            if segmented:
                words = chunk_words.split()
            else:
                words = chunk_words  # actually chars
            for word_or_char in words:
                if word_or_char == ' ':
                    continue
                pair = word_or_char + ' ' + 'O\n'
                pair_list.append(pair)
    return pair_list


def outputWithTagScheme(input_list, label, tagScheme="BMES"):
    output_list = []
    list_length = len(input_list)
    if tagScheme == "BMES":
        if list_length == 1:
            pair = input_list[0] + ' ' + 'S-' + label + '\n'
            output_list.append(pair)
        else:
            for idx in range(list_length):
                if idx == 0:
                    pair = input_list[idx] + ' ' + 'B-' + label + '\n'
                elif idx == list_length - 1:
                    pair = input_list[idx] + ' ' + 'E-' + label + '\n'
                else:
                    pair = input_list[idx] + ' ' + 'M-' + label + '\n'
                output_list.append(pair)
    else:
        for idx in range(list_length):
            if idx == 0:
                pair = input_list[idx] + ' ' + 'B-' + label + '\n'
            else:
                pair = input_list[idx] + ' ' + 'I-' + label + '\n'
            output_list.append(pair)
    return output_list


def removeRecommendContent(content, recommendRe=r'\[\$.*?\#.*?\*\](?!\#)'):
    output_content = ""
    last_match_end = 0
    for match in re.finditer(recommendRe, content):
        matched = content[match.span()[0]:match.span()[1]]
        words = matched.strip('[$]').split("#")[0]
        output_content += content[last_match_end:match.span()[0]] + words
        last_match_end = match.span()[1]
    output_content += content[last_match_end:]
    return output_content


def main():
    print("YEDDA launched!")
    print("OS:", platform.system())
    root = Tk()
    width, height = 1300, 700
    x = max((root.winfo_screenwidth() - width) // 2, 0)
    y = max((root.winfo_screenheight() - height) // 2, 0)
    root.geometry(f'{width}x{height}+{x}+{y}')
    app = Application(root)
    app.setFont(17)
    root.mainloop()


if __name__ == '__main__':
    main()