Overhaul content processing

No dummy main categories, resolve incomplete references.
Following the spec more closely, identifying apppriate categories.
Don't consider application base library type as category.

contrib/conv_cat.py

@@ -8,12 +8,11 @@
 
 debug = True
 
-edges = collections.defaultdict(lambda: set())
 paths = collections.defaultdict(lambda: list())
 hints = dict()
 
 # a few special ones, those from the spec are added from table input
-main_cats = {"Accessibility", "Screensavers", "WINE", "Other"}
+main_cats = set() # "Accessibility", "Screensavers", "WINE"}
 
 
 def add_edges(key :str, multicand :list):
@@ -32,6 +31,8 @@ def add_edges(key :str, multicand :list):
         main_cats.add(row[0].strip())
         hints[row[0]] = row[1]
 
+if debug:
+    print(f"main cats: {main_cats}", file=sys.stderr)
 
 with open('Additional_Categories.csv', newline='') as csvfile:
     rdr = csv.reader(csvfile, dialect='unix')
@@ -40,55 +41,42 @@ def add_edges(key :str, multicand :list):
         assert(len(row) == 3)
         if " " in row[0]:  # the header
             continue
-        multicand = list(map(lambda s: s.strip(), re.split(r'\sor\s', row[2])))
-        add_edges(row[0], multicand)
+        # we don't care about "based on foo library", that is not a real section
+        if "Application based on" in row[1]:
+            continue
         hints[row[0]] = row[1]
+        for m in re.split(r'\s+or\s+', row[2]):
+            m = m.strip()
+            paths[row[0]].append(list(m.split(';')))
+
+# let's fixup references which are not leading to some main category directly
+resolved = False
+while not resolved:
+    resolved = True
+    for k, v in paths.items():
+        for w in v:
+            cat = w[0]
+            if not cat or cat in main_cats:
+                continue
+            if cat not in paths:
+                print(f"Warning, cannot resolve {cat}", file=sys.stderr)
+                continue
+            for more in paths[cat]:
+                w[:0] = more
+                resolved = False
 
+print(f"{paths}", file=sys.stderr)
 
-#print(edges, file=sys.stderr)
+groupedByLength = collections.defaultdict(lambda: list())
+for k, v in paths.items():
+    for l in v:
+        groupedByLength[1+len(l)].append(l + [k])
 
+groupedByLength[1] = list(map(lambda x: [x], main_cats))
 
-def xxx():
-    """
-    Probably BS, trying to expand the graph, but fails on equally named nodes.
-    :return:
-    """
-    global edges
-    keyz = list(edges.keys())
-    for k in keyz:
-        temp = list(edges[k])
-        newset = set()
-        for vv in temp:
-            xp = vv.split(';')
-            if len(xp) == 1:
-                continue
-            #print(f"multi cand? {k} <-> {vv}")
-            newset.add(xp[-1])
-            chain = list(xp)
-            chain.append(k)
-            print(f"chain? {chain}", file=sys.stderr)
-            while len(chain) > 1:
-                add_edges(chain[-1], [chain[-2]])
-                chain.pop(-1)
-            edges[k] = newset
-
-
-for k, v in edges.items():
-    for vv in v:
-        #print(f"XX: {vv}")
-        xp = [k] + list(reversed(vv.strip().split(';')))
-        if not xp[-1]:
-            xp[-1] = "Other"
-        if xp[-1] not in main_cats:
-            xp = xp + ["Other"]
-        if debug: print(f"{len(xp)} -> {xp}", file=sys.stderr)
-        paths[len(xp)].append(xp)
-
-paths[1] = list(map(lambda x: [x], main_cats))
-
-if debug: print(paths, file=sys.stderr)
-
-keysByLen = list(reversed(sorted(paths.keys())))
+print(f"{groupedByLength}", file=sys.stderr)
+
+#keysByLen = list(reversed(sorted(paths.keys())))
 
 print(f"""/**
  * WARNING: this file is autogenerated! Any change might be overwritten!
@@ -107,23 +95,22 @@ def xxx():
 using t_menu_path_table = std::initializer_list<t_menu_path>;
 using t_menu_path_table_list = std::initializer_list<t_menu_path_table>;
 
-#define MENU_DEPTH_MAX {keysByLen[0]}
-
 constexpr t_menu_path_table_list valid_paths = {{""")
 
-for k in keysByLen:
+for k in reversed(sorted(groupedByLength.keys())):
     print(f"\n\t// menu locations of depth {k}\n\t{{")
-    byFirst = sorted(paths[k], key=lambda l: l[0])
-    for v in byFirst:
+    ways = groupedByLength[k]
+    for v in sorted(ways, key=lambda x: x[-1]):
         print("\t\t{")
-        for t in v:
+        for t in reversed(v):
             print("// TRANSLATORS: This is a SHORT category menu name from freedesktop.org. Please add compact punctuation if needed but no double-quotes! Hint for the content inside: " + hints.get(t, t))
-            print("\t\t\tN_(\"" + t + "\"),")
+            if t:
+                print("\t\t\tN_(\"" + t + "\"),")
+            else:
+                print("\t\t\t\"" + t + "\",")
         print("\t\t},")
     print("\t},")
 
 print(f"""}};
 
-#define SIZE_OF_MENU_TABLE {len(keysByLen)}
-
 #endif // FDO_GEN_MENU_STRUCTURE_H""")

src/fdomenu.cc

@@ -37,6 +37,7 @@
 #include <set>
 #include <string>
 #include <utility> // For std::move
+#include <vector>
 
 #include <functional>
 #include <initializer_list>
@@ -89,10 +90,11 @@ char *terminal_command;
 char *terminal_option;
 
 // global defaults and helpers
-static string ICON_FOLDER("folder");
+static const string ICON_FOLDER("folder"), OTH("Other");
 string indent_hint("");
 // use this to dump optional comment data, deque is pointer-stable
 deque<string> comment_pool;
+set<string> valid_main_cats;
 
 /*
  * Certain parts borrowed from apt-cacher-ng by its autor, either from older
@@ -143,6 +145,10 @@ template <typename T> struct lessByDerefAdaptor {
     bool operator()(const T *a, const T *b) { return *a < *b; }
 };
 
+template <typename T> const T &iback(const initializer_list<T> &q) {
+    return *(q.end() - 1);
+}
+
 /**
  * Basic base implementation of a reference-counted class
  */
@@ -720,34 +726,83 @@ struct MenuNode {
     }
 };
 
+const auto lessFirstStr = [](const t_menu_path &a, const t_menu_path &b) {
+    return strcmp(*a.begin(), *b.begin()) < 0;
+};
+
 void MenuNode::sink_in(DesktopFilePtr pDf) {
 
-    static const auto lessFirstStr = [](const t_menu_path &a,
-                                        const t_menu_path &b) {
-        return strcmp(*a.begin(), *b.begin()) < 0;
+    // XXX: use plain pointers, with length, or string_view? Probably alost
+    // worthless because SSO applies in the vast majority of cases
+    static vector<string> main_cats, sub_cats;
+    main_cats.clear();
+    sub_cats.clear();
+
+    for (tSplitWalk split(pDf->Categories, ";"); split.Next();) {
+        string k(split);
+        auto hit = valid_main_cats.find(k);
+        if (hit != valid_main_cats.end()) {
+            DBGMSG("mcat: " << k);
+            main_cats.push_back(std::move(k));
+        } else {
+            DBGMSG("scat: " << k);
+            sub_cats.push_back(std::move(k));
+        }
+    }
+    bool added_somewhere = false;
+    auto install = [&pDf, &added_somewhere](MenuNode *cur) {
+        if (!cur)
+            return;
+        added_somewhere = true;
+        cur->apps.emplace(pDf->GetNamePtr(), AppEntry(pDf));
     };
 
-    auto add_sub_menues = [&](const t_menu_path &mp) {
-        MenuNode *cur = this;
-        for (auto it = mp.end() - 1; it >= mp.begin(); --it)
-            cur = &cur->submenus[(*it && **it) ? *it : "Other"];
+    auto add_sub_menues = [&](const t_menu_path &mp, MenuNode *cur,
+                              int roffset = -1) {
+        auto itLast = mp.end() + roffset;
+        for (auto it = itLast; it >= mp.begin(); --it)
+            cur = &cur->submenus[*it];
         return cur;
     };
 
-    for (tSplitWalk split(pDf->Categories, ";"); split.Next();) {
-        auto cat = split.str();
-        t_menu_path refval = {cat.c_str()};
-
-        // maybe start main cats only
-        for (auto w = no_sub_cats ? (valid_paths.end() - 1)
-                                  : valid_paths.begin();
-             w != valid_paths.end(); ++w) {
+    for (const auto &sk : sub_cats) {
+        t_menu_path refval = {sk.c_str()};
 
+        for (auto w = valid_paths.begin(); w != valid_paths.end(); ++w) {
             auto rng =
                 std::equal_range(w->begin(), w->end(), refval, lessFirstStr);
-            for (auto it = rng.first; it != rng.second; ++it)
-                (*add_sub_menues(*it))
-                    .apps.emplace(pDf->GetNamePtr(), AppEntry(pDf));
+            for (auto iPath = rng.first; iPath != rng.second; ++iPath) {
+                string mk = iback(*iPath);
+                if (no_sub_cats) {
+                    if (!mk.empty())
+                        install(&submenus[mk]);
+                } else if (!mk.empty()) {
+                    // easy case, path to the exact main category is known
+                    install(add_sub_menues(*iPath, this, -1));
+                } else {
+                    // apply to all cats as per spec
+                    if (main_cats.empty())
+                        install(add_sub_menues(*iPath, &submenus[OTH], -2));
+                    else {
+
+                        for (const auto &any_mk : main_cats) {
+                            install(
+                                add_sub_menues(*iPath, &submenus[any_mk], -2));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // catch-all
+    if (!added_somewhere) {
+
+        if (main_cats.empty())
+            install(&submenus[OTH]);
+        else {
+            for (const auto &mk : main_cats)
+                install(&submenus[mk]);
         }
     }
 }
@@ -1129,6 +1184,9 @@ int main(int argc, char **argv) {
         deadline_all = now + std::chrono::milliseconds(b);
     }
 
+    for (const auto &p : *(valid_paths.end() - 1))
+        valid_main_cats.insert(*p.begin());
+
     auto justLang = string(msglang ? msglang : "");
     justLang = justLang.substr(0, justLang.find('.'));