src/dict/dictionary: better filter mecab and deconjugator duplicates

Better filter queries that are produced by both the deconjugator and
MeCab. Previously MeCab results were generally not filtered if they were
duplicates of deconjugator results. This would lead to duplicate search
results where some would have conjugation information and some would
not. This is generally undesirable. This changes the behavior to filter
out MeCab results that are duplicates of Deconjugator results when
compiled with MECAB_SUPPORT.

The filtering algorithm is not trivial. This required adding metadata
indicating the source alogrithm that each came from. If this had not
been done, it would have been difficult to discriminate MeCab queries
from Exact queries. The algorithm consists of finding every deconj
string that the deconjugator came up with. Then it tosses any MeCab
queries that have a deconj string that the deconjugator already found.
This is unforunteately not behavior that can be implemented with
std::unqiue easily.

src/dict/deconjugationquerygenerator.cpp

@@ -79,10 +79,11 @@ std::vector<SearchQuery> DeconjugationQueryGenerator::generateQueries(
         else
         {
             result.emplace_back(SearchQuery{
+                SearchQuery::Source::deconj,
                 info.base,
                 info.conjugated,
                 { rule },
-                info.derivationDisplay
+                info.derivationDisplay,
             });
         }
 

src/dict/dictionary.cpp

@@ -281,6 +281,38 @@ void Dictionary::sortQueries(std::vector<SearchQuery> &queries)
 
 void Dictionary::filterDuplicates(std::vector<SearchQuery> &queries)
 {
+#ifdef MECAB_SUPPORT
+    /* Remove all duplicates that both MeCab and the deconjugator got.
+     * Prefer deconjugator results over MeCab. */
+    QSet<QString> deconjQueries;
+    for (const SearchQuery &query : queries)
+    {
+        if (query.source == SearchQuery::Source::deconj)
+        {
+            deconjQueries.insert(query.deconj);
+        }
+    }
+    queries.erase(
+        std::remove_if(
+            std::begin(queries), std::end(queries),
+            [&deconjQueries] (const SearchQuery &query) -> bool
+            {
+                switch (query.source)
+                {
+                    case SearchQuery::Source::mecab:
+                        return deconjQueries.contains(query.deconj);
+
+                    case SearchQuery::Source::deconj:
+                    case SearchQuery::Source::exact:
+                        return false;
+                }
+                return false;
+            }
+        ),
+        std::end(queries)
+    );
+#endif // MECAB_SUPPORT
+
     auto last = std::unique(
         std::begin(queries), std::end(queries),
         [] (const SearchQuery &lhs, const SearchQuery &rhs) -> bool

src/dict/exactquerygenerator.cpp

@@ -31,6 +31,7 @@ std::vector<SearchQuery> ExactQueryGenerator::generateQueries(
         SearchQuery sq;
         sq.deconj = query;
         sq.surface = query;
+        sq.source = SearchQuery::Source::exact;
         queries.emplace_back(std::move(sq));
         query.chop(1);
     }

src/dict/mecabquerygenerator.cpp

@@ -153,6 +153,7 @@ MeCabQueryGenerator::generateQueriesHelper(const MeCab::Node *node)
             query.deconj = deconj;
             query.surface = surface;
             query.surfaceClean = surfaceClean;
+            query.source = SearchQuery::Source::mecab;
             queries.emplace_back(std::move(query));
         }
 

src/dict/searchquery.h

@@ -30,6 +30,19 @@
  */
 struct SearchQuery
 {
+    /**
+     * Enumeration of SearchQuery sources.
+     */
+    enum class Source
+    {
+        exact,
+        deconj,
+        mecab,
+    };
+
+    /* The query algorithm this query comes from */
+    Source source;
+
     /* The deconjugated string */
     QString deconj;
 

src/util/constants.h

@@ -151,7 +151,7 @@ namespace Constants
 
 #ifdef MECAB_SUPPORT
                 constexpr const char *MECAB_IPADIC = "ipadic-matcher";
-                constexpr bool MECAB_IPADIC_DEFAULT = false;
+                constexpr bool MECAB_IPADIC_DEFAULT = true;
 #endif // MECAB_SUPPORT
             }