Add a needle check script, remove some unused needles

I call this...The @lruzicka Catcher!

It's a script that checks for needles that aren't actually used
anywhere. It also checks for cases where we have a needle JSON
file but no image, or an image file but no JSON file (and wipes
one case of the latter). It also adds a run of the script to tox
so we get it in CI.

You could make this script a lot more elaborate if you like, by
being fancier about parsing the test code and templates, but I
don't think it's really warranted, I think it just needs to be
'good enough'. It's not the end of the world if it misses the
odd thing or the whitelisting goes stale.

Quite a lot of the removed needles are remnants of different
approaches to app start/stop testing which weren't caught in the
initial PR review. The short-name partitioning ones are odd; they
were introduced in the commit that moved needles into subdirs,
but at least some of them don't actually appear to be moves. They
may have been non-tracked files Josef had lying around that got
into the commit by mistake, or they may just be old needles we
really used at some point but aren't using any more.
reclaim_space_second_partition was introduced as part of the
shrink test (along with reclaim_space_first_partition) but was
never actually used by that test - I guess, again, the test got
re-written during review but we forgot to remove the needle. We
rejigged user creation to use tab presses not a needle match a
while back, which made user_creation_password_input unnecessary.
The various cockpit_updates_* needles are I think remnants of
rewrites of the cockpit update tests that again were missed in
PR review, the tests as merged never used them.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

check-needles.py

@@ -0,0 +1,146 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2020 Red Hat
+#
+# This file is part of os-autoinst-distri-fedora.
+#
+# os-autoinst-distri-fedora is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation, either version 2 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Author: Adam Williamson <awilliam@redhat.com>
+
+"""This is a check script which checks for unused needles. If none of
+the tags a needle declares is referenced in the tests, it is
+considered unused.
+"""
+
+import glob
+import json
+import os
+import re
+import sys
+
+NEEDLEPATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "needles")
+TESTSPATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests")
+LIBPATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "lib")
+# these don't account for escaping, but I don't think we're ever going
+# to have an escaped quotation mark in a needle tag
+DOUBLEQUOTERE = re.compile('"(.*?)"')
+SINGLEQUOTERE = re.compile("'(.*?)'")
+
+# first we're gonna build a big list of all string literals
+testpaths = glob.glob(f"{TESTSPATH}/**/*.pm", recursive=True)
+testpaths.extend(glob.glob(f"{LIBPATH}/**/*.pm", recursive=True))
+testliterals = []
+for testpath in testpaths:
+    # skip if it's a symlink
+    if os.path.islink(testpath):
+        continue
+    # otherwise, scan it for string literals
+    with open(testpath, "r") as testfh:
+        testtext = testfh.read()
+    for match in DOUBLEQUOTERE.finditer(testtext):
+        testliterals.append(match[1])
+    for match in SINGLEQUOTERE.finditer(testtext):
+        testliterals.append(match[1])
+
+# now let's do some whitelisting, for awkward cases where we know that
+# we concatenate string literals and stuff
+# versioned backgrounds and release IDs
+for rel in range(30, 100):
+    testliterals.append(f"{rel}_background")
+    testliterals.append(f"version_{rel}_ident")
+# anaconda id needles, using tell_source
+for source in ("workstation", "generic", "server"):
+    testliterals.append(f"leftbar_{source}")
+    testliterals.append(f"topbar_{source}")
+# keyboard layout switching, using desktop_switch_layout
+for environment in ("anaconda", "gnome"):
+    for layout in ("native", "ascii"):
+        testliterals.append(f"{environment}_layout_{layout}")
+# package set selection, using get_var('PACKAGE_SET')
+for pkgset in ("kde", "workstation", "minimal"):
+    testliterals.append(f"anaconda_{pkgset}_highlighted")
+    testliterals.append(f"anaconda_{pkgset}_selected")
+# partitioning stuff, there's a bunch of this, all in anaconda.pm
+# multiple things use this
+for part in ("swap", "root"):
+    testliterals.append(f"anaconda_part_select_{part}")
+# select_disks
+for num in range(1, 10):
+    testliterals.append(f"anaconda_install_destination_select_disk_{num}")
+# custom_scheme_select
+for scheme in ("standard", "lvmthin", "btrfs"):
+    testliterals.append(f"anaconda_part_scheme_{scheme}")
+# custom_blivet_add_partition
+for dtype in ("lvm", "lvmthin", "raid"):
+    testliterals.append(f"anaconda_blivet_part_devicetype_{dtype}")
+for fsys in ("ext3", "ext4", "xfs", "btrfs", "ppc_prep_boot", "swap", "efi_filesystem"):
+    testliterals.append(f"anaconda_blivet_part_fs_{fsys}")
+    testliterals.append(f"anaconda_blivet_part_fs_{fsys}_selected")
+# this is variable-y in custom_change_type but we only actually have
+# one value
+testliterals.append("anaconda_part_device_type_raid")
+# custom_change_fs
+for fsys in ("ext3", "xfs"):
+    testliterals.append(f"anaconda_part_fs_{fsys}")
+    testliterals.append(f"anaconda_part_fs_{fsys}_selected")
+# variable-y in custom_change_device but we only have one value
+testliterals.append("anaconda_part_device_sda")
+
+# retcode tracker
+ret = 0
+
+# now let's scan our needles
+unused = []
+noimg = []
+noneedle = []
+
+needlepaths = glob.glob(f"{NEEDLEPATH}/**/*.json", recursive=True)
+for needlepath in needlepaths:
+    # check we have a matching image file
+    imgpath = needlepath.replace(".json", ".png")
+    if not os.path.exists(imgpath):
+        noimg.append(needlepath)
+    with open(needlepath, "r") as needlefh:
+        needlejson = json.load(needlefh)
+    if any(tag in testliterals for tag in needlejson["tags"]):
+        continue
+    unused.append(needlepath)
+
+# reverse check, for images without a needle file
+imgpaths = glob.glob(f"{NEEDLEPATH}/**/*.png", recursive=True)
+for imgpath in imgpaths:
+    needlepath = imgpath.replace(".png", ".json")
+    if not os.path.exists(needlepath):
+        noneedle.append(imgpath)
+
+if unused:
+    ret += 1
+    print("Unused needle(s) found!")
+    for needle in unused:
+        print(needle)
+
+if noimg:
+    ret += 2
+    print("Needle(s) without image(s) found!")
+    for needle in noimg:
+        print(needle)
+
+if noneedle:
+    ret += 4
+    print("Image(s) without needle(s) found!")
+    for img in noneedle:
+        print(img)
+
+sys.exit(ret)

needles/anaconda/install_destination/reclaim_space_second_partition.json

@@ -1,16 +0,0 @@
-{
-  "area": [
-    {
-      "xpos": 11,
-      "ypos": 301,
-      "width": 75,
-      "height": 25,
-      "type": "match"
-    }
-  ],
-  "tags": [
-    "anaconda_install_destination_reclaim_space_second_partition",
-    "ENV-DISTRI-fedora",
-    "ENV-FLAVOR-server"
-  ]
-}

needles/anaconda/install_process/french/user_creation_password_input_french.json

@@ -1,18 +0,0 @@
-{
-    "area": [
-        {
-            "height": 25,
-            "type": "match",
-            "width": 268,
-            "xpos": 250,
-            "ypos": 285
-        }
-    ],
-    "properties": [],
-    "tags": [
-        "anaconda_user_creation_password_input",
-        "ENV-DISTRI-fedora",
-        "LANGUAGE-french",
-        "ENV-FLAVOR-server"
-    ]
-}

needles/anaconda/install_process/russian/user_creation_password_input_russian.json

@@ -1,16 +0,0 @@
-{
-  "tags": [
-    "LANGUAGE-russian",
-    "anaconda_user_creation_password_input"
-  ],
-  "area": [
-    {
-      "xpos": 287,
-      "ypos": 279,
-      "width": 251,
-      "height": 18,
-      "type": "match"
-    }
-  ],
-  "properties": []
-}

needles/anaconda/install_process/user_creation_password_input-cantarell21.json

@@ -1,18 +0,0 @@
-{
-  "area": [
-    {
-      "height": 25,
-      "type": "match",
-      "width": 243,
-      "xpos": 260,
-      "ypos": 276
-    }
-  ],
-  "properties": [],
-  "tags": [
-    "anaconda_user_creation_password_input",
-    "ENV-DISTRI-fedora",
-    "LANGUAGE-english",
-    "ENV-FLAVOR-server"
-  ]
-}

needles/anaconda/partitioning/add_mountpoint.json

@@ -1,18 +0,0 @@
-{
-  "properties": [],
-  "area": [
-    {
-      "xpos": 545,
-      "ypos": 461,
-      "width": 118,
-      "height": 26,
-      "type": "match"
-    }
-  ],
-  "tags": [
-    "anaconda_part_add_mountpoint",
-    "ENV-DISTRI-fedora",
-    "LANGUAGE-english",
-    "ENV-FLAVOR-server"
-  ]
-}

needles/anaconda/partitioning/desired_capacity.json

@@ -1,18 +0,0 @@
-{
-  "area": [
-    {
-      "xpos": 351,
-      "ypos": 406,
-      "width": 304,
-      "height": 21,
-      "type": "match"
-    }
-  ],
-  "tags": [
-    "anaconda_part_desired_capacity",
-    "ENV-DISTRI-fedora",
-    "LANGUAGE-english",
-    "ENV-FLAVOR-server"
-  ],
-  "properties": []
-}

needles/anaconda/partitioning/list_box_boot.json

@@ -1,17 +0,0 @@
-{
-  "tags": [
-    "anaconda_part_list_box_boot",
-    "ENV-DISTRI-fedora",
-    "ENV-FLAVOR-server"
-  ],
-  "area": [
-    {
-      "xpos": 471,
-      "ypos": 433,
-      "width": 63,
-      "height": 28,
-      "type": "match"
-    }
-  ],
-  "properties": []
-}

needles/anaconda/partitioning/list_box_button.json

@@ -1,17 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "anaconda_part_list_box_button",
-    "ENV-DISTRI-fedora",
-    "ENV-FLAVOR-server"
-  ],
-  "area": [
-    {
-      "xpos": 640,
-      "ypos": 372,
-      "width": 27,
-      "height": 22,
-      "type": "match"
-    }
-  ]
-}

needles/anaconda/partitioning/list_box_root.json

@@ -1,17 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "anaconda_part_list_box_root",
-    "ENV-DISTRI-fedora",
-    "ENV-FLAVOR-server"
-  ],
-  "area": [
-    {
-      "xpos": 472,
-      "ypos": 405,
-      "width": 66,
-      "height": 26,
-      "type": "match"
-    }
-  ]
-}

needles/anaconda/partitioning/list_box_swap.json

@@ -1,17 +0,0 @@
-{
-    "area": [
-        {
-            "height": 24,
-            "type": "match",
-            "width": 91,
-            "xpos": 472,
-            "ypos": 528
-        }
-    ],
-    "properties": [],
-    "tags": [
-        "anaconda_part_list_box_swap",
-        "ENV-DISTRI-fedora",
-        "ENV-FLAVOR-server"
-    ]
-}

needles/anaconda/partitioning/plus_button.json

@@ -1,17 +0,0 @@
-{
-    "area": [
-        {
-            "height": 31,
-            "type": "match",
-            "width": 35,
-            "xpos": 32,
-            "ypos": 619
-        }
-    ],
-    "properties": [],
-    "tags": [
-        "anaconda_part_plus_button",
-        "ENV-DISTRI-fedora",
-        "ENV-FLAVOR-server"
-    ]
-}

needles/anaconda/partitioning/scheme_lvm.json

@@ -1,16 +0,0 @@
-{
-    "area": [
-        {
-            "height": 20,
-            "type": "match",
-            "width": 35,
-            "xpos": 60,
-            "ypos": 300
-        }
-    ],
-    "tags": [
-        "anaconda_part_scheme_lvm",
-        "ENV-DISTRI-fedora",
-        "ENV-FLAVOR-server"
-    ]
-}

needles/cockpit/cockpit_updates_available.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "cockpit_updates_available"
-  ],
-  "area": [
-    {
-      "xpos": 264,
-      "ypos": 259,
-      "width": 185,
-      "height": 28,
-      "type": "match"
-    }
-  ]
-}

needles/cockpit/cockpit_updates_console_update.json

@@ -1,15 +0,0 @@
-{
-  "area": [
-    {
-      "xpos": 304,
-      "ypos": 309,
-      "width": 143,
-      "height": 17,
-      "type": "match"
-    }
-  ],
-  "properties": [],
-  "tags": [
-    "cockpit_updates_console_update"
-  ]
-}

needles/cockpit/cockpit_updates_restart_confirm.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "cockpit_updates_restart_confirm"
-  ],
-  "area": [
-    {
-      "xpos": 576,
-      "ypos": 260,
-      "width": 46,
-      "height": 15,
-      "type": "match"
-    }
-  ]
-}

needles/gnome/apps/apps_menu_documents.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_menu_documents"
-  ],
-  "area": [
-    {
-      "xpos": 491,
-      "ypos": 298,
-      "width": 36,
-      "height": 47,
-      "type": "match"
-    }
-  ]
-}

needles/gnome/apps/apps_menu_settings-20200316.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_menu_settings"
-  ],
-  "area": [
-    {
-      "xpos": 756,
-      "ypos": 446,
-      "width": 61,
-      "height": 43,
-      "type": "match"
-    }
-  ]
-}

needles/gnome/apps/apps_menu_settings.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_menu_settings"
-  ],
-  "area": [
-    {
-      "xpos": 634,
-      "ypos": 583,
-      "width": 37,
-      "height": 37,
-      "type": "match"
-    }
-  ]
-}

needles/gnome/apps/apps_menu_terminal-20190128.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_menu_terminal"
-  ],
-  "area": [
-    {
-      "xpos": 207,
-      "height": 32,
-      "width": 45,
-      "type": "match",
-      "ypos": 588
-    }
-  ]
-}

needles/gnome/apps/apps_menu_terminal-20200316.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_menu_terminal"
-  ],
-  "area": [
-    {
-      "xpos": 574,
-      "ypos": 567,
-      "width": 59,
-      "height": 38,
-      "type": "match"
-    }
-  ]
-}

needles/gnome/apps/apps_menu_terminal.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_menu_terminal"
-  ],
-  "area": [
-    {
-      "xpos": 209,
-      "ypos": 591,
-      "width": 45,
-      "height": 32,
-      "type": "match"
-    }
-  ]
-}

needles/gnome/apps/apps_run_clocks_access.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_run_clocks_access"
-  ],
-  "area": [
-    {
-      "xpos": 602,
-      "ypos": 478,
-      "width": 105,
-      "height": 21,
-      "type": "ocr"
-    }
-  ]
-}

needles/gnome/apps/apps_run_documents.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_run_documents"
-  ],
-  "area": [
-    {
-      "xpos": 412,
-      "ypos": 39,
-      "width": 79,
-      "height": 23,
-      "type": "match"
-    }
-  ]
-}

needles/gnome/apps/apps_run_nautilus.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_run_nautilus"
-  ],
-  "area": [
-    {
-      "xpos": 157,
-      "ypos": 86,
-      "width": 67,
-      "height": 23,
-      "type": "ocr"
-    }
-  ]
-}

needles/gnome/apps/apps_run_settings.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_run_settings"
-  ],
-  "area": [
-    {
-      "xpos": 104,
-      "ypos": 41,
-      "width": 61,
-      "height": 21,
-      "type": "ocr"
-    }
-  ]
-}

needles/gnome/apps/apps_run_weather_access.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_run_weather_access"
-  ],
-  "area": [
-    {
-      "xpos": 601,
-      "ypos": 474,
-      "width": 105,
-      "height": 27,
-      "type": "ocr"
-    }
-  ]
-}

needles/gnome/apps/apps_sub_utilities.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_sub_utilities"
-  ],
-  "area": [
-    {
-      "xpos": 616,
-      "ypos": 159,
-      "width": 71,
-      "height": 58,
-      "type": "match"
-    }
-  ]
-}

needles/gnome/apps/apps_utilities.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "apps_utilities"
-  ],
-  "area": [
-    {
-      "xpos": 463,
-      "ypos": 154,
-      "width": 100,
-      "height": 37,
-      "type": "match"
-    }
-  ]
-}

needles/kde/apps/firewall_requires.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "firewall_requires"
-  ],
-  "area": [
-    {
-      "xpos": 595,
-      "ypos": 422,
-      "width": 19,
-      "height": 16,
-      "type": "match"
-    }
-  ]
-}

needles/kde/apps/kamoso_runs.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "kamoso_runs"
-  ],
-  "area": [
-    {
-      "xpos": 322,
-      "ypos": 9,
-      "width": 58,
-      "height": 15,
-      "type": "match"
-    }
-  ]
-}

needles/kde/apps/menu_favorites.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "menu_favorites"
-  ],
-  "area": [
-    {
-      "xpos": 37,
-      "ypos": 654,
-      "width": 36,
-      "height": 31,
-      "type": "match"
-    }
-  ]
-}

needles/kde/apps/users_runs.json

@@ -1,15 +0,0 @@
-{
-  "properties": [],
-  "tags": [
-    "users_runs"
-  ],
-  "area": [
-    {
-      "xpos": 468,
-      "ypos": 116,
-      "width": 96,
-      "height": 19,
-      "type": "match"
-    }
-  ]
-}

tox.ini

@@ -13,6 +13,7 @@ deps =
 
 commands=
     ./fifloader.py --clean templates.fif.json templates-updates.fif.json
+    ./check-needles.py
     py.test unittests/
     py.test --cov-report term-missing --cov-report xml --cov fifloader unittests/
     diff-cover coverage.xml --fail-under=90