TextProcessor/libsrc/libTargomanTextProcessor/TextProcessor.h at main · Targoman/TextProcessor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
/******************************************************************************
 * Targoman: A robust Machine Translation framework               *
 *                                                                            *
 * Copyright 2014-2018 by ITRC <http://itrc.ac.ir>                            *
 *                                                                            *
 * This file is part of Targoman.                                             *
 *                                                                            *
 * Targoman is free software: you can redistribute it and/or modify           *
 * it under the terms of the GNU Lesser General Public License as published   *
 * by the Free Software Foundation, either version 3 of the License, or       *
 * (at your option) any later version.                                        *
 *                                                                            *
 * Targoman is distributed in the hope that it will be useful,                *
 * but WITHOUT ANY WARRANTY; without even the implied warranty of             *
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
 * GNU Lesser General Public License for more details.                        *
 * You should have received a copy of the GNU Lesser General Public License   *
 * along with Targoman. If not, see <http://www.gnu.org/licenses/>.           *
 *                                                                            *
 ******************************************************************************/
/**
 * @author S. Mohammad M. Ziabary <ziabary@targoman.com>
 * @author Behrooz Vedadian <vedadian@targoman.com>
 * @author Saeed Torabzadeh <saeed.torabzadeh@targoman.com>
 */

#ifndef TARGOMAN_NLPLIBS_TARGOMANTP_TEXTPROCESSOR_H
#define TARGOMAN_NLPLIBS_TARGOMANTP_TEXTPROCESSOR_H

#include <QString>
#include "libTargomanCommon/Macros.h"
#include "libTargomanCommon/exTargomanBase.h"
#include "libTargomanCommon/Logger.h"
#include "libTargomanCommon/Types.h"
#include "ISO639.h" //From https://github.com/softnhard/ISO639

namespace Targoman {
namespace NLPLibs {

TARGOMAN_DEFINE_ENHANCED_ENUM(enuTextTags,
                              Number,
                              SpecialNumber,
                              Email,
                              URL,
                              Abbreviation,
                              OrderedListItem,
                              Time,
                              Date,
                              Ordinals,
                              Symbol
                              );

TARGOMAN_ADD_EXCEPTION_HANDLER(exTextProcessor, Targoman::Common::exTargomanBase);

struct stuIXMLReplacement{
    QRegularExpression SearchRegExp;
    QString AfterString;

    stuIXMLReplacement(const QRegularExpression& _searchRegExp = QRegularExpression(),
                       const QString& _afterString = QString()):
        SearchRegExp(_searchRegExp),
        AfterString(_afterString)
    {}
};

class TargomanTextProcessor
{
public:
    struct stuConfigs{
        QString NormalizationFile;
        QString AbbreviationsFile;
        QString SpellCorrectorBaseConfigPath;
        QHash<QString, QVariantHash> SpellCorrectorLanguageBasedConfigs;
    };

public:
    static inline TargomanTextProcessor& instance(){
        static TargomanTextProcessor* Instance = nullptr;
        return *(Q_LIKELY(Instance) ? Instance : (Instance = new TargomanTextProcessor));
    }

    bool init(const stuConfigs &_configs);
    bool init(QSharedPointer<QSettings> _configSettings = QSharedPointer<QSettings>());

    QString text2IXML(const QString& _inStr,
                      INOUT bool &_spellCorrected,
                      const QString& _lang = "",
                      quint32 _lineNo = 0,
                      bool _interactive = true,
                      bool _useSpellCorrector = true,
                      QList<enuTextTags::Type> _removingTags = QList<enuTextTags::Type>(),
                      QList<stuIXMLReplacement> _replacements = QList<stuIXMLReplacement>()) const;

    QString ixml2Text(const QString& _ixml,
                      bool _detokenize = true,
                      bool _hinidiDigits = false,
                      bool _arabicPunctuations = false,
                      bool _breakSentences = false) const;

    inline QString normalizeText(const QString _input,
                                 bool _interactive,
                                 const QString &_lang) const{
        bool SpellCorrected;
        return this->normalizeText(_input, SpellCorrected, _interactive, _lang);
    }

    QString normalizeText(const QString _input,
                          INOUT bool &_spellCorrected,
                          bool _interactive = false,
                          const QString& _lang = "") const;

private:
    TargomanTextProcessor();
    Q_DISABLE_COPY(TargomanTextProcessor)
};


}
}

#endif // TARGOMAN_NLPLIBS_TARGOMANTP_TEXTPROCESSOR_H