Merge remote-tracking branch 'upstream/master' into lotus-march2023

lib/python/qmk/cli/__init__.py

@@ -34,13 +34,12 @@ subcommands = [
     'qmk.cli.bux',
     'qmk.cli.c2json',
     'qmk.cli.cd',
-    'qmk.cli.cformat',
     'qmk.cli.chibios.confmigrate',
     'qmk.cli.clean',
     'qmk.cli.compile',
     'qmk.cli.docs',
     'qmk.cli.doctor',
-    'qmk.cli.fileformat',
+    'qmk.cli.find',
     'qmk.cli.flash',
     'qmk.cli.format.c',
     'qmk.cli.format.json',
@@ -57,9 +56,11 @@ subcommands = [
     'qmk.cli.generate.keyboard_c',
     'qmk.cli.generate.keyboard_h',
     'qmk.cli.generate.keycodes',
+    'qmk.cli.generate.keycodes_tests',
     'qmk.cli.generate.rgb_breathe_table',
     'qmk.cli.generate.rules_mk',
     'qmk.cli.generate.version_h',
+    'qmk.cli.git.submodule',
     'qmk.cli.hello',
     'qmk.cli.import.kbfirmware',
     'qmk.cli.import.keyboard',
@@ -67,16 +68,15 @@ subcommands = [
     'qmk.cli.info',
     'qmk.cli.json2c',
     'qmk.cli.lint',
+    'qmk.cli.kle2json',
     'qmk.cli.list.keyboards',
     'qmk.cli.list.keymaps',
     'qmk.cli.list.layouts',
-    'qmk.cli.kle2json',
     'qmk.cli.mass_compile',
-    'qmk.cli.multibuild',
+    'qmk.cli.migrate',
     'qmk.cli.new.keyboard',
     'qmk.cli.new.keymap',
     'qmk.cli.painter',
-    'qmk.cli.pyformat',
     'qmk.cli.pytest',
     'qmk.cli.via2json',
 ]

lib/python/qmk/cli/cformat.py

@@ -1,28 +0,0 @@
-"""Point people to the new command name.
-"""
-import sys
-from pathlib import Path
-
-from milc import cli
-
-
-@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.")
-@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
-@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.')
-@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.')
-@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.')
-@cli.subcommand('Pointer to the new command name: qmk format-c.', hidden=True)
-def cformat(cli):
-    """Pointer to the new command name: qmk format-c.
-    """
-    cli.log.warning('"qmk cformat" has been renamed to "qmk format-c". Please use the new command in the future.')
-    argv = [sys.executable, *sys.argv]
-    argv[argv.index('cformat')] = 'format-c'
-    script_path = Path(argv[1])
-    script_path_exe = Path(f'{argv[1]}.exe')
-
-    if not script_path.exists() and script_path_exe.exists():
-        # For reasons I don't understand ".exe" is stripped from the script name on windows.
-        argv[1] = str(script_path_exe)
-
-    return cli.run(argv, capture_output=False).returncode

lib/python/qmk/cli/compile.py

@@ -10,7 +10,17 @@ import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.keymap import keymap_completer
+from qmk.keymap import keymap_completer, locate_keymap
+
+
+def _is_keymap_target(keyboard, keymap):
+    if keymap == 'all':
+        return True
+
+    if locate_keymap(keyboard, keymap):
+        return True
+
+    return False
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
@@ -43,6 +53,11 @@ def compile(cli):
 
     elif cli.config.compile.keyboard and cli.config.compile.keymap:
         # Generate the make command for a specific keyboard/keymap.
+        if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
+            cli.log.error('Invalid keymap argument.')
+            cli.print_help()
+            return False
+
         if cli.args.clean:
             commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
         commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))

lib/python/qmk/cli/doctor/check.py

@@ -3,7 +3,7 @@
 from enum import Enum
 import re
 import shutil
-from subprocess import DEVNULL
+from subprocess import DEVNULL, TimeoutExpired
 
 from milc import cli
 from qmk import submodules
@@ -41,9 +41,8 @@ def _parse_gcc_version(version):
 def _check_arm_gcc_version():
     """Returns True if the arm-none-eabi-gcc version is not known to cause problems.
     """
-    if 'output' in ESSENTIAL_BINARIES['arm-none-eabi-gcc']:
-        version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip()
-        cli.log.info('Found arm-none-eabi-gcc version %s', version_number)
+    version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip()
+    cli.log.info('Found arm-none-eabi-gcc version %s', version_number)
 
     return CheckStatus.OK  # Right now all known arm versions are ok
 
@@ -51,44 +50,37 @@ def _check_arm_gcc_version():
 def _check_avr_gcc_version():
     """Returns True if the avr-gcc version is not known to cause problems.
     """
-    rc = CheckStatus.ERROR
-    if 'output' in ESSENTIAL_BINARIES['avr-gcc']:
-        version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip()
+    version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip()
+    cli.log.info('Found avr-gcc version %s', version_number)
 
-        cli.log.info('Found avr-gcc version %s', version_number)
-        rc = CheckStatus.OK
+    parsed_version = _parse_gcc_version(version_number)
+    if parsed_version['major'] > 8:
+        cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.')
+        return CheckStatus.WARNING
 
-        parsed_version = _parse_gcc_version(version_number)
-        if parsed_version['major'] > 8:
-            cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.')
-            rc = CheckStatus.WARNING
-
-    return rc
+    return CheckStatus.OK
 
 
 def _check_avrdude_version():
-    if 'output' in ESSENTIAL_BINARIES['avrdude']:
-        last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2]
-        version_number = last_line.split()[2][:-1]
-        cli.log.info('Found avrdude version %s', version_number)
+    last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2]
+    version_number = last_line.split()[2][:-1]
+    cli.log.info('Found avrdude version %s', version_number)
 
     return CheckStatus.OK
 
 
 def _check_dfu_util_version():
-    if 'output' in ESSENTIAL_BINARIES['dfu-util']:
-        first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0]
-        version_number = first_line.split()[1]
-        cli.log.info('Found dfu-util version %s', version_number)
+    first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0]
+    version_number = first_line.split()[1]
+    cli.log.info('Found dfu-util version %s', version_number)
 
     return CheckStatus.OK
 
 
 def _check_dfu_programmer_version():
-    if 'output' in ESSENTIAL_BINARIES['dfu-programmer']:
-        first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0]
-        version_number = first_line.split()[1]
-        cli.log.info('Found dfu-programmer version %s', version_number)
+    first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0]
+    version_number = first_line.split()[1]
+    cli.log.info('Found dfu-programmer version %s', version_number)
 
     return CheckStatus.OK
 
@@ -96,11 +88,16 @@ def _check_dfu_programmer_version():
 def check_binaries():
     """Iterates through ESSENTIAL_BINARIES and tests them.
     """
-    ok = True
+    ok = CheckStatus.OK
 
     for binary in sorted(ESSENTIAL_BINARIES):
-        if not is_executable(binary):
-            ok = False
+        try:
+            if not is_executable(binary):
+                ok = CheckStatus.ERROR
+        except TimeoutExpired:
+            cli.log.debug('Timeout checking %s', binary)
+            if ok != CheckStatus.ERROR:
+                ok = CheckStatus.WARNING
 
     return ok
 
@@ -108,8 +105,22 @@ def check_binaries():
 def check_binary_versions():
     """Check the versions of ESSENTIAL_BINARIES
     """
+    checks = {
+        'arm-none-eabi-gcc': _check_arm_gcc_version,
+        'avr-gcc': _check_avr_gcc_version,
+        'avrdude': _check_avrdude_version,
+        'dfu-util': _check_dfu_util_version,
+        'dfu-programmer': _check_dfu_programmer_version,
+    }
+
     versions = []
-    for check in (_check_arm_gcc_version, _check_avr_gcc_version, _check_avrdude_version, _check_dfu_util_version, _check_dfu_programmer_version):
+    for binary in sorted(ESSENTIAL_BINARIES):
+        if 'output' not in ESSENTIAL_BINARIES[binary]:
+            cli.log.warning('Unknown version for %s', binary)
+            versions.append(CheckStatus.WARNING)
+            continue
+
+        check = checks[binary]
         versions.append(check())
     return versions
 
@@ -119,10 +130,8 @@ def check_submodules():
     """
     for submodule in submodules.status().values():
         if submodule['status'] is None:
-            cli.log.error('Submodule %s has not yet been cloned!', submodule['name'])
             return CheckStatus.ERROR
         elif not submodule['status']:
-            cli.log.warning('Submodule %s is not up to date!', submodule['name'])
             return CheckStatus.WARNING
 
     return CheckStatus.OK
@@ -149,3 +158,21 @@ def is_executable(command):
 
     cli.log.error("{fg_red}Can't run `%s %s`", command, version_arg)
     return False
+
+
+def release_info(file='/etc/os-release'):
+    """Parse release info to dict
+    """
+    ret = {}
+    try:
+        with open(file) as f:
+            for line in f:
+                if '=' in line:
+                    key, value = map(str.strip, line.split('=', 1))
+                    if value.startswith('"') and value.endswith('"'):
+                        value = value[1:-1]
+                    ret[key] = value
+    except (PermissionError, FileNotFoundError):
+        pass
+
+    return ret

lib/python/qmk/cli/doctor/linux.py

@@ -7,7 +7,11 @@ from pathlib import Path
 from milc import cli
 
 from qmk.constants import QMK_FIRMWARE, BOOTLOADER_VIDS_PIDS
-from .check import CheckStatus
+from .check import CheckStatus, release_info
+
+
+def _is_wsl():
+    return 'microsoft' in platform.uname().release.lower()
 
 
 def _udev_rule(vid, pid=None, *args):
@@ -78,10 +82,13 @@ def check_udev_rules():
 
         # Collect all rules from the config files
         for rule_file in udev_rules:
-            for line in rule_file.read_text(encoding='utf-8').split('\n'):
-                line = line.strip()
-                if not line.startswith("#") and len(line):
-                    current_rules.add(line)
+            try:
+                for line in rule_file.read_text(encoding='utf-8').split('\n'):
+                    line = line.strip()
+                    if not line.startswith("#") and len(line):
+                        current_rules.add(line)
+            except PermissionError:
+                cli.log.debug("Failed to read: %s", rule_file)
 
         # Check if the desired rules are among the currently present rules
         for bootloader, rules in desired_rules.items():
@@ -127,17 +134,22 @@ def check_modem_manager():
 def os_test_linux():
     """Run the Linux specific tests.
     """
-    # Don't bother with udev on WSL, for now
-    if 'microsoft' in platform.uname().release.lower():
-        cli.log.info("Detected {fg_cyan}Linux (WSL){fg_reset}.")
+    info = release_info()
+    release_id = info.get('PRETTY_NAME', info.get('ID', 'Unknown'))
+    plat = 'WSL, ' if _is_wsl() else ''
 
+    cli.log.info(f"Detected {{fg_cyan}}Linux ({plat}{release_id}){{fg_reset}}.")
+
+    # Don't bother with udev on WSL, for now
+    if _is_wsl():
         # https://github.com/microsoft/WSL/issues/4197
         if QMK_FIRMWARE.as_posix().startswith("/mnt"):
             cli.log.warning("I/O performance on /mnt may be extremely slow.")
             return CheckStatus.WARNING
 
-        return CheckStatus.OK
     else:
-        cli.log.info("Detected {fg_cyan}Linux{fg_reset}.")
+        rc = check_udev_rules()
+        if rc != CheckStatus.OK:
+            return rc
 
-        return check_udev_rules()
+    return CheckStatus.OK

lib/python/qmk/cli/doctor/main.py

@@ -119,13 +119,15 @@ def doctor(cli):
     # Make sure the basic CLI tools we need are available and can be executed.
     bin_ok = check_binaries()
 
-    if not bin_ok:
+    if bin_ok == CheckStatus.ERROR:
         if yesno('Would you like to install dependencies?', default=True):
             cli.run(['util/qmk_install.sh', '-y'], stdin=DEVNULL, capture_output=False)
             bin_ok = check_binaries()
 
-    if bin_ok:
+    if bin_ok == CheckStatus.OK:
         cli.log.info('All dependencies are installed.')
+    elif bin_ok == CheckStatus.WARNING:
+        cli.log.warning('Issues encountered while checking dependencies.')
     else:
         status = CheckStatus.ERROR
 
@@ -142,7 +144,7 @@ def doctor(cli):
     if sub_ok == CheckStatus.OK:
         cli.log.info('Submodules are up to date.')
     else:
-        if yesno('Would you like to clone the submodules?', default=True):
+        if git_check_repo() and yesno('Would you like to clone the submodules?', default=True):
             submodules.update()
             sub_ok = check_submodules()
 

lib/python/qmk/cli/doctor/windows.py

@@ -2,7 +2,7 @@ import platform
 
 from milc import cli
 
-from .check import CheckStatus
+from .check import CheckStatus, release_info
 
 
 def os_test_windows():
@@ -11,4 +11,10 @@ def os_test_windows():
     win32_ver = platform.win32_ver()
     cli.log.info("Detected {fg_cyan}Windows %s (%s){fg_reset}.", win32_ver[0], win32_ver[1])
 
+    # MSYS really does not like "/" files - resolve manually
+    file = cli.run(['cygpath', '-m', '/etc/qmk-release']).stdout.strip()
+    qmk_distro_version = release_info(file).get('VERSION', None)
+    if qmk_distro_version:
+        cli.log.info('QMK MSYS version: %s', qmk_distro_version)
+
     return CheckStatus.OK

lib/python/qmk/cli/fileformat.py

@@ -1,23 +0,0 @@
-"""Point people to the new command name.
-"""
-import sys
-from pathlib import Path
-
-from milc import cli
-
-
-@cli.subcommand('Pointer to the new command name: qmk format-text.', hidden=True)
-def fileformat(cli):
-    """Pointer to the new command name: qmk format-text.
-    """
-    cli.log.warning('"qmk fileformat" has been renamed to "qmk format-text". Please use the new command in the future.')
-    argv = [sys.executable, *sys.argv]
-    argv[argv.index('fileformat')] = 'format-text'
-    script_path = Path(argv[1])
-    script_path_exe = Path(f'{argv[1]}.exe')
-
-    if not script_path.exists() and script_path_exe.exists():
-        # For reasons I don't understand ".exe" is stripped from the script name on windows.
-        argv[1] = str(script_path_exe)
-
-    return cli.run(argv, capture_output=False).returncode

lib/python/qmk/cli/find.py

@@ -0,0 +1,23 @@
+"""Command to search through all keyboards and keymaps for a given search criteria.
+"""
+from milc import cli
+from qmk.search import search_keymap_targets
+
+
+@cli.argument(
+    '-f',
+    '--filter',
+    arg_only=True,
+    action='append',
+    default=[],
+    help=  # noqa: `format-python` and `pytest` don't agree here.
+    "Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
+)
+@cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
+@cli.subcommand('Find builds which match supplied search criteria.')
+def find(cli):
+    """Search through all keyboards and keymaps for a given search criteria.
+    """
+    targets = search_keymap_targets(cli.args.keymap, cli.args.filter)
+    for target in targets:
+        print(f'{target[0]}:{target[1]}')

lib/python/qmk/cli/flash.py

@@ -11,12 +11,24 @@ import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.keymap import keymap_completer, locate_keymap
 from qmk.flashers import flasher
 
 
-def print_bootloader_help():
+def _is_keymap_target(keyboard, keymap):
+    if keymap == 'all':
+        return True
+
+    if locate_keymap(keyboard, keymap):
+        return True
+
+    return False
+
+
+def _list_bootloaders():
     """Prints the available bootloaders listed in docs.qmk.fm.
     """
+    cli.print_help()
     cli.log.info('Here are the available bootloaders:')
     cli.echo('\tavrdude')
     cli.echo('\tbootloadhid')
@@ -36,14 +48,29 @@ def print_bootloader_help():
     cli.echo('\tuf2-split-left')
     cli.echo('\tuf2-split-right')
     cli.echo('For more info, visit https://docs.qmk.fm/#/flashing')
+    return False
+
+
+def _flash_binary(filename, mcu):
+    """Try to flash binary firmware
+    """
+    cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n')
+    try:
+        err, msg = flasher(mcu, filename)
+        if err:
+            cli.log.error(msg)
+            return False
+    except KeyboardInterrupt:
+        cli.log.info('Ctrl-C was pressed, exiting...')
+    return True
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.')
 @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
 @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
 @cli.argument('-m', '--mcu', help='The MCU name. Required for HalfKay, HID, USBAspLoader and ISP flashing.')
-@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
+@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@@ -56,30 +83,17 @@ def flash(cli):
 
     If a binary firmware is supplied, try to flash that.
 
-    If a Configurator JSON export is supplied this command will create a new keymap. Keymap and Keyboard arguments
-    will be ignored.
+    If a Configurator export is supplied this command will create a new keymap, overwriting an existing keymap if one exists.
 
-    If no file is supplied, keymap and keyboard are expected.
+    If a keyboard and keymap are provided this command will build a firmware based on that.
 
     If bootloader is omitted the make system will use the configured bootloader for that keyboard.
     """
-    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex']:
-        # Try to flash binary firmware
-        cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n')
-        try:
-            err, msg = flasher(cli.args.mcu, cli.args.filename)
-            if err:
-                cli.log.error(msg)
-                return False
-        except KeyboardInterrupt:
-            cli.log.info('Ctrl-C was pressed, exiting...')
-        return True
+    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
+        return _flash_binary(cli.args.filename, cli.args.mcu)
 
     if cli.args.bootloaders:
-        # Provide usage and list bootloaders
-        cli.print_help()
-        print_bootloader_help()
-        return False
+        return _list_bootloaders()
 
     # Build the environment vars
     envs = build_environment(cli.args.env)
@@ -94,6 +108,11 @@ def flash(cli):
 
     elif cli.config.flash.keyboard and cli.config.flash.keymap:
         # Generate the make command for a specific keyboard/keymap.
+        if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
+            cli.log.error('Invalid keymap argument.')
+            cli.print_help()
+            return False
+
         if cli.args.clean:
             commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))
         commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs))

lib/python/qmk/cli/generate/api.py

@@ -10,8 +10,9 @@ from qmk.datetime import current_datetime
 from qmk.info import info_json
 from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
+from qmk.keymap import list_keymaps
 from qmk.keyboard import find_readme, list_keyboards
-from qmk.keycodes import load_spec, list_versions
+from qmk.keycodes import load_spec, list_versions, list_languages
 
 DATA_PATH = Path('data')
 TEMPLATE_PATH = DATA_PATH / 'templates/api/'
@@ -42,7 +43,14 @@ def _resolve_keycode_specs(output_folder):
         overall = load_spec(version)
 
         output_file = output_folder / f'constants/keycodes_{version}.json'
-        output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
+        output_file.write_text(json.dumps(overall), encoding='utf-8')
+
+    for lang in list_languages():
+        for version in list_versions(lang):
+            overall = load_spec(version, lang)
+
+            output_file = output_folder / f'constants/keycodes_{lang}_{version}.json'
+            output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
 
     # Purge files consumed by 'load_spec'
     shutil.rmtree(output_folder / 'constants/keycodes/')
@@ -56,7 +64,7 @@ def _filtered_copy(src, dst):
         data = json_load(src)
 
         dst = dst.with_suffix('.json')
-        dst.write_text(json.dumps(data, indent=4), encoding='utf-8')
+        dst.write_text(json.dumps(data), encoding='utf-8')
         return dst
 
     return shutil.copy2(src, dst)
@@ -103,24 +111,44 @@ def generate_api(cli):
 
     # Generate and write keyboard specific JSON files
     for keyboard_name in keyboard_list:
-        kb_all[keyboard_name] = info_json(keyboard_name)
+        kb_json = info_json(keyboard_name)
+        kb_all[keyboard_name] = kb_json
+
         keyboard_dir = v1_dir / 'keyboards' / keyboard_name
         keyboard_info = keyboard_dir / 'info.json'
         keyboard_readme = keyboard_dir / 'readme.md'
         keyboard_readme_src = find_readme(keyboard_name)
 
+        # Populate the list of JSON keymaps
+        for keymap in list_keymaps(keyboard_name, c=False, fullpath=True):
+            kb_json['keymaps'][keymap.name] = {
+                # TODO: deprecate 'url' as consumer needs to know its potentially hjson
+                'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json',
+
+                # Instead consumer should grab from API and not repo directly
+                'path': (keymap / 'keymap.json').as_posix(),
+            }
+
         keyboard_dir.mkdir(parents=True, exist_ok=True)
-        keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_all[keyboard_name]}})
+        keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_json}})
         if not cli.args.dry_run:
-            keyboard_info.write_text(keyboard_json)
+            keyboard_info.write_text(keyboard_json, encoding='utf-8')
             cli.log.debug('Wrote file %s', keyboard_info)
 
             if keyboard_readme_src:
                 shutil.copyfile(keyboard_readme_src, keyboard_readme)
                 cli.log.debug('Copied %s -> %s', keyboard_readme_src, keyboard_readme)
 
-        if 'usb' in kb_all[keyboard_name]:
-            usb = kb_all[keyboard_name]['usb']
+            # resolve keymaps as json
+            for keymap in kb_json['keymaps']:
+                keymap_hjson = kb_json['keymaps'][keymap]['path']
+                keymap_json = v1_dir / keymap_hjson
+                keymap_json.parent.mkdir(parents=True, exist_ok=True)
+                keymap_json.write_text(json.dumps(json_load(Path(keymap_hjson))), encoding='utf-8')
+                cli.log.debug('Wrote keymap %s', keymap_json)
+
+        if 'usb' in kb_json:
+            usb = kb_json['usb']
 
             if 'vid' in usb and usb['vid'] not in usb_list:
                 usb_list[usb['vid']] = {}
@@ -153,9 +181,9 @@ def generate_api(cli):
     constants_metadata_json = json.dumps({'last_updated': current_datetime(), 'constants': _list_constants(v1_dir)})
 
     if not cli.args.dry_run:
-        keyboard_all_file.write_text(keyboard_all_json)
-        usb_file.write_text(usb_json)
-        keyboard_list_file.write_text(keyboard_list_json)
-        keyboard_aliases_file.write_text(keyboard_aliases_json)
-        keyboard_metadata_file.write_text(keyboard_metadata_json)
-        constants_metadata_file.write_text(constants_metadata_json)
+        keyboard_all_file.write_text(keyboard_all_json, encoding='utf-8')
+        usb_file.write_text(usb_json, encoding='utf-8')
+        keyboard_list_file.write_text(keyboard_list_json, encoding='utf-8')
+        keyboard_aliases_file.write_text(keyboard_aliases_json, encoding='utf-8')
+        keyboard_metadata_file.write_text(keyboard_metadata_json, encoding='utf-8')
+        constants_metadata_file.write_text(constants_metadata_json, encoding='utf-8')

lib/python/qmk/cli/generate/keyboard_c.py

@@ -25,17 +25,17 @@ def _gen_led_config(info_data):
     if not config_type:
         return lines
 
-    matrix = [['NO_LED'] * cols for i in range(rows)]
+    matrix = [['NO_LED'] * cols for _ in range(rows)]
     pos = []
     flags = []
 
-    led_config = info_data[config_type]['layout']
-    for index, item in enumerate(led_config, start=0):
-        if 'matrix' in item:
-            (x, y) = item['matrix']
-            matrix[x][y] = str(index)
-        pos.append(f'{{ {item.get("x", 0)},{item.get("y", 0)} }}')
-        flags.append(str(item.get('flags', 0)))
+    led_layout = info_data[config_type]['layout']
+    for index, led_data in enumerate(led_layout):
+        if 'matrix' in led_data:
+            row, col = led_data['matrix']
+            matrix[row][col] = str(index)
+        pos.append(f'{{{led_data.get("x", 0)}, {led_data.get("y", 0)}}}')
+        flags.append(str(led_data.get('flags', 0)))
 
     if config_type == 'rgb_matrix':
         lines.append('#ifdef RGB_MATRIX_ENABLE')
@@ -47,10 +47,10 @@ def _gen_led_config(info_data):
     lines.append('__attribute__ ((weak)) led_config_t g_led_config = {')
     lines.append('  {')
     for line in matrix:
-        lines.append(f'    {{ {",".join(line)} }},')
+        lines.append(f'    {{ {", ".join(line)} }},')
     lines.append('  },')
-    lines.append(f'  {{ {",".join(pos)} }},')
-    lines.append(f'  {{ {",".join(flags)} }},')
+    lines.append(f'  {{ {", ".join(pos)} }},')
+    lines.append(f'  {{ {", ".join(flags)} }},')
     lines.append('};')
     lines.append('#endif')
 

lib/python/qmk/cli/generate/keyboard_h.py

@@ -25,32 +25,31 @@ def _generate_layouts(keyboard):
     row_num = kb_info_json['matrix_size']['rows']
 
     lines = []
-    for layout_name in kb_info_json['layouts']:
-        if kb_info_json['layouts'][layout_name]['c_macro']:
+    for layout_name, layout_data in kb_info_json['layouts'].items():
+        if layout_data['c_macro']:
             continue
 
-        if 'matrix' not in kb_info_json['layouts'][layout_name]['layout'][0]:
-            cli.log.debug(f'{keyboard}/{layout_name}: No matrix data!')
+        if not all('matrix' in key_data for key_data in layout_data['layout']):
+            cli.log.debug(f'{keyboard}/{layout_name}: No or incomplete matrix data!')
             continue
 
         layout_keys = []
-        layout_matrix = [['KC_NO' for i in range(col_num)] for i in range(row_num)]
+        layout_matrix = [['KC_NO'] * col_num for _ in range(row_num)]
 
-        for i, key in enumerate(kb_info_json['layouts'][layout_name]['layout']):
-            row = key['matrix'][0]
-            col = key['matrix'][1]
-            identifier = 'k%s%s' % (ROW_LETTERS[row], COL_LETTERS[col])
+        for index, key_data in enumerate(layout_data['layout']):
+            row, col = key_data['matrix']
+            identifier = f'k{ROW_LETTERS[row]}{COL_LETTERS[col]}'
 
             try:
                 layout_matrix[row][col] = identifier
                 layout_keys.append(identifier)
             except IndexError:
-                key_name = key.get('label', identifier)
-                cli.log.error(f'Matrix data out of bounds for layout {layout_name} at index {i} ({key_name}): [{row}, {col}]')
+                key_name = key_data.get('label', identifier)
+                cli.log.error(f'{keyboard}/{layout_name}: Matrix data out of bounds at index {index} ({key_name}): [{row}, {col}]')
                 return []
 
         lines.append('')
-        lines.append('#define %s(%s) {\\' % (layout_name, ', '.join(layout_keys)))
+        lines.append(f'#define {layout_name}({", ".join(layout_keys)}) {{ \\')
 
         rows = ', \\\n'.join(['\t {' + ', '.join(row) + '}' for row in layout_matrix])
         rows += ' \\'

lib/python/qmk/cli/generate/keycodes.py

@@ -8,6 +8,34 @@ from qmk.path import normpath
 from qmk.keycodes import load_spec
 
 
+def _translate_group(group):
+    """Fix up any issues with badly chosen values
+    """
+    if group == 'modifiers':
+        return 'modifier'
+    if group == 'media':
+        return 'consumer'
+    return group
+
+
+def _render_key(key):
+    width = 7
+    if 'S(' in key:
+        width += len('S()')
+    if 'A(' in key:
+        width += len('A()')
+    if 'RCTL(' in key:
+        width += len('RCTL()')
+    if 'ALGR(' in key:
+        width += len('ALGR()')
+    return key.ljust(width)
+
+
+def _render_label(label):
+    label = label.replace("\\", "(backslash)")
+    return label
+
+
 def _generate_ranges(lines, keycodes):
     lines.append('')
     lines.append('enum qk_keycode_ranges {')
@@ -64,7 +92,24 @@ def _generate_helpers(lines, keycodes):
     for group, codes in temp.items():
         lo = keycodes["keycodes"][f'0x{codes[0]:04X}']['key']
         hi = keycodes["keycodes"][f'0x{codes[1]:04X}']['key']
-        lines.append(f'#define IS_{ group.upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})')
+        lines.append(f'#define IS_{ _translate_group(group).upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})')
+
+
+def _generate_aliases(lines, keycodes):
+    lines.append('')
+    lines.append('// Aliases')
+    for key, value in keycodes["aliases"].items():
+        define = _render_key(value.get("key"))
+        val = _render_key(key)
+        if 'label' in value:
+            lines.append(f'#define {define} {val} // {_render_label(value.get("label"))}')
+        else:
+            lines.append(f'#define {define} {val}')
+
+    lines.append('')
+    for key, value in keycodes["aliases"].items():
+        for alias in value.get("aliases", []):
+            lines.append(f'#define {alias} {value.get("key")}')
 
 
 @cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
@@ -86,3 +131,23 @@ def generate_keycodes(cli):
 
     # Show the results
     dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)
+
+
+@cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
+@cli.argument('-l', '--lang', arg_only=True, required=True, help='Language of keycodes to generate.')
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.subcommand('Used by the make system to generate keymap_{lang}.h from keycodes_{lang}_{version}.json', hidden=True)
+def generate_keycode_extras(cli):
+    """Generates the header file.
+    """
+
+    # Build the header file.
+    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '#include "keymap.h"', '// clang-format off']
+
+    keycodes = load_spec(cli.args.version, cli.args.lang)
+
+    _generate_aliases(keycodes_h_lines, keycodes)
+
+    # Show the results
+    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)

lib/python/qmk/cli/generate/keycodes_tests.py

@@ -0,0 +1,39 @@
+"""Used by the make system to generate a keycode lookup table from keycodes_{version}.json
+"""
+from milc import cli
+
+from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
+from qmk.commands import dump_lines
+from qmk.path import normpath
+from qmk.keycodes import load_spec
+
+
+def _generate_defines(lines, keycodes):
+    lines.append('')
+    lines.append('std::map<uint16_t, std::string> KEYCODE_ID_TABLE = {')
+    for key, value in keycodes["keycodes"].items():
+        lines.append(f'    {{{value.get("key")}, "{value.get("key")}"}},')
+    lines.append('};')
+
+
+@cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.subcommand('Used by the make system to generate a keycode lookup table from keycodes_{version}.json', hidden=True)
+def generate_keycodes_tests(cli):
+    """Generates a keycode to identifier lookup table for unit test output.
+    """
+
+    # Build the keycodes.h file.
+    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '// clang-format off']
+    keycodes_h_lines.append('extern "C" {\n#include <keycode.h>\n}')
+    keycodes_h_lines.append('#include <map>')
+    keycodes_h_lines.append('#include <string>')
+    keycodes_h_lines.append('#include <cstdint>')
+
+    keycodes = load_spec(cli.args.version)
+
+    _generate_defines(keycodes_h_lines, keycodes)
+
+    # Show the results
+    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)

lib/python/qmk/cli/generate/version_h.py

@@ -6,7 +6,7 @@ from milc import cli
 
 from qmk.path import normpath
 from qmk.commands import dump_lines
-from qmk.git import git_get_version
+from qmk.git import git_get_qmk_hash, git_get_version, git_is_dirty
 from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
 
 TIME_FMT = '%Y-%m-%d-%H:%M:%S'
@@ -29,23 +29,30 @@ def generate_version_h(cli):
         current_time = strftime(TIME_FMT)
 
     if cli.args.skip_git:
+        git_dirty = False
         git_version = "NA"
+        git_qmk_hash = "NA"
         chibios_version = "NA"
         chibios_contrib_version = "NA"
     else:
+        git_dirty = git_is_dirty()
         git_version = git_get_version() or current_time
+        git_qmk_hash = git_get_qmk_hash() or "Unknown"
         chibios_version = git_get_version("chibios", "os") or current_time
         chibios_contrib_version = git_get_version("chibios-contrib", "os") or current_time
 
     # Build the version.h file.
     version_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once']
 
-    version_h_lines.append(f"""
+    version_h_lines.append(
+        f"""
 #define QMK_VERSION "{git_version}"
 #define QMK_BUILDDATE "{current_time}"
+#define QMK_GIT_HASH  "{git_qmk_hash}{'*' if git_dirty else ''}"
 #define CHIBIOS_VERSION "{chibios_version}"
 #define CHIBIOS_CONTRIB_VERSION "{chibios_contrib_version}"
-""")
+"""
+    )
 
     # Show the results
     dump_lines(cli.args.output, version_h_lines, cli.args.quiet)

lib/python/qmk/cli/git/submodule.py

@@ -0,0 +1,38 @@
+import shutil
+
+from milc import cli
+
+from qmk.path import normpath
+from qmk import submodules
+
+REMOVE_DIRS = [
+    'lib/ugfx',
+    'lib/pico-sdk',
+    'lib/chibios-contrib/ext/mcux-sdk',
+    'lib/lvgl',
+]
+
+
+@cli.argument('--check', arg_only=True, action='store_true', help='Check if the submodules are dirty, and display a warning if they are.')
+@cli.argument('--sync', arg_only=True, action='store_true', help='Shallow clone any missing submodules.')
+@cli.subcommand('Git Submodule actions.')
+def git_submodule(cli):
+    """Git Submodule actions
+    """
+    if cli.args.check:
+        return all(item['status'] for item in submodules.status().values())
+
+    if cli.args.sync:
+        cli.run(['git', 'submodule', 'sync', '--recursive'])
+        for name, item in submodules.status().items():
+            if item['status'] is None:
+                cli.run(['git', 'submodule', 'update', '--depth=50', '--init', name], capture_output=False)
+        return True
+
+    for folder in REMOVE_DIRS:
+        if normpath(folder).is_dir():
+            print(f"Removing '{folder}'")
+            shutil.rmtree(folder)
+
+    cli.run(['git', 'submodule', 'sync', '--recursive'], capture_output=False)
+    cli.run(['git', 'submodule', 'update', '--init', '--recursive', '--progress'], capture_output=False)

lib/python/qmk/cli/list/keyboards.py

@@ -5,9 +5,10 @@ from milc import cli
 import qmk.keyboard
 
 
+@cli.argument('--no-resolve-defaults', arg_only=True, action='store_false', help='Ignore any "DEFAULT_FOLDER" within keyboards rules.mk')
 @cli.subcommand("List the keyboards currently defined within QMK")
 def list_keyboards(cli):
     """List the keyboards currently defined within QMK
     """
-    for keyboard_name in qmk.keyboard.list_keyboards():
+    for keyboard_name in qmk.keyboard.list_keyboards(cli.args.no_resolve_defaults):
         print(keyboard_name)

lib/python/qmk/cli/mass_compile.py

@@ -2,52 +2,14 @@
 
 This will compile everything in parallel, for testing purposes.
 """
-import fnmatch
-import logging
-import multiprocessing
 import os
-import re
 from pathlib import Path
 from subprocess import DEVNULL
-from dotty_dict import dotty
 from milc import cli
 
 from qmk.constants import QMK_FIRMWARE
 from qmk.commands import _find_make, get_make_parallel_args
-from qmk.info import keymap_json
-import qmk.keyboard
-import qmk.keymap
-
-
-def _set_log_level(level):
-    cli.acquire_lock()
-    old = cli.log_level
-    cli.log_level = level
-    cli.log.setLevel(level)
-    logging.root.setLevel(level)
-    cli.release_lock()
-    return old
-
-
-def _all_keymaps(keyboard):
-    old = _set_log_level(logging.CRITICAL)
-    keymaps = qmk.keymap.list_keymaps(keyboard)
-    _set_log_level(old)
-    return (keyboard, keymaps)
-
-
-def _keymap_exists(keyboard, keymap):
-    old = _set_log_level(logging.CRITICAL)
-    ret = keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
-    _set_log_level(old)
-    return ret
-
-
-def _load_keymap_info(keyboard, keymap):
-    old = _set_log_level(logging.CRITICAL)
-    ret = (keyboard, keymap, keymap_json(keyboard, keymap))
-    _set_log_level(old)
-    return ret
+from qmk.search import search_keymap_targets
 
 
 @cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
@@ -60,7 +22,7 @@ def _load_keymap_info(keyboard, keymap):
     action='append',
     default=[],
     help=  # noqa: `format-python` and `pytest` don't agree here.
-    "Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the format 'features.rgblight=true'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
+    "Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
 )
 @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@@ -75,49 +37,7 @@ def mass_compile(cli):
     builddir = Path(QMK_FIRMWARE) / '.build'
     makefile = builddir / 'parallel_kb_builds.mk'
 
-    targets = []
-
-    with multiprocessing.Pool() as pool:
-        cli.log.info(f'Retrieving list of keyboards with keymap "{cli.args.keymap}"...')
-        target_list = []
-        if cli.args.keymap == 'all':
-            kb_to_kms = pool.map(_all_keymaps, qmk.keyboard.list_keyboards())
-            for targets in kb_to_kms:
-                keyboard = targets[0]
-                keymaps = targets[1]
-                target_list.extend([(keyboard, keymap) for keymap in keymaps])
-        else:
-            target_list = [(kb, cli.args.keymap) for kb in filter(lambda kb: kb is not None, pool.starmap(_keymap_exists, [(kb, cli.args.keymap) for kb in qmk.keyboard.list_keyboards()]))]
-
-        if len(cli.args.filter) == 0:
-            targets = target_list
-        else:
-            cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
-            valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in pool.starmap(_load_keymap_info, target_list)]
-
-            filter_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
-            for filter_txt in cli.args.filter:
-                f = filter_re.match(filter_txt)
-                if f is not None:
-                    key = f.group('key')
-                    value = f.group('value')
-                    cli.log.info(f'Filtering on condition ("{key}" == "{value}")...')
-
-                    def _make_filter(k, v):
-                        expr = fnmatch.translate(v)
-                        rule = re.compile(expr, re.IGNORECASE)
-
-                        def f(e):
-                            lhs = e[2].get(k)
-                            lhs = str(False if lhs is None else lhs)
-                            return rule.search(lhs) is not None
-
-                        return f
-
-                    valid_keymaps = filter(_make_filter(key, value), valid_keymaps)
-
-            targets = [(e[0], e[1]) for e in valid_keymaps]
-
+    targets = search_keymap_targets(cli.args.keymap, cli.args.filter)
     if len(targets) == 0:
         return
 
@@ -134,7 +54,7 @@ all: {keyboard_safe}_{keymap_name}_binary
 {keyboard_safe}_{keymap_name}_binary:
 	@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}.{keymap_name}" || true
 	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}"
-	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false {' '.join(cli.args.env)} \\
+	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(cli.args.env)} \\
 		>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" 2>&1 \\
 		|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
 	@{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\

lib/python/qmk/cli/migrate.py

@@ -0,0 +1,81 @@
+"""Migrate keyboard configuration to "Data Driven"
+"""
+import json
+from pathlib import Path
+from dotty_dict import dotty
+
+from milc import cli
+
+from qmk.keyboard import keyboard_completer, keyboard_folder, resolve_keyboard
+from qmk.info import info_json, find_info_json
+from qmk.json_encoders import InfoJSONEncoder
+from qmk.json_schema import json_load
+
+
+def _candidate_files(keyboard):
+    kb_dir = Path(resolve_keyboard(keyboard))
+
+    cur_dir = Path('keyboards')
+    files = []
+    for dir in kb_dir.parts:
+        cur_dir = cur_dir / dir
+        files.append(cur_dir / 'config.h')
+        files.append(cur_dir / 'rules.mk')
+
+    return [file for file in files if file.exists()]
+
+
+@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the performed migrations based on the supplied value. Supported format is 'KEY' located from 'data/mappings'. May be passed multiple times.")
+@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='The keyboard\'s name')
+@cli.subcommand('Migrate keyboard config to "Data Driven".', hidden=True)
+def migrate(cli):
+    """Migrate keyboard configuration to "Data Driven"
+    """
+    # Merge mappings as we do not care to where "KEY" is found just that its removed
+    info_config_map = json_load(Path('data/mappings/info_config.hjson'))
+    info_rules_map = json_load(Path('data/mappings/info_rules.hjson'))
+    info_map = {**info_config_map, **info_rules_map}
+
+    # Parse target info.json which will receive updates
+    target_info = Path(find_info_json(cli.args.keyboard)[0])
+    info_data = dotty(json_load(target_info))
+
+    # Already parsed used for updates
+    kb_info_json = dotty(info_json(cli.args.keyboard))
+
+    # List of candidate files
+    files = _candidate_files(cli.args.keyboard)
+
+    # Filter down keys if requested
+    keys = info_map.keys()
+    if cli.args.filter:
+        keys = list(set(keys) & set(cli.args.filter))
+
+    cli.log.info(f'{{fg_green}}Migrating keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}}.{{fg_reset}}')
+
+    # Start migration
+    for file in files:
+        cli.log.info(f'  Migrating file {file}')
+        file_contents = file.read_text(encoding='utf-8').split('\n')
+        for key in keys:
+            for num, line in enumerate(file_contents):
+                if line.startswith(f'{key} =') or line.startswith(f'#define {key} '):
+                    cli.log.info(f'    Migrating {key}...')
+
+                    while line.rstrip().endswith('\\'):
+                        file_contents.pop(num)
+                        line = file_contents[num]
+                    file_contents.pop(num)
+
+                    update_key = info_map[key]["info_key"]
+                    if update_key in kb_info_json:
+                        info_data[update_key] = kb_info_json[update_key]
+
+        file.write_text('\n'.join(file_contents), encoding='utf-8')
+
+    # Finally write out updated info.json
+    cli.log.info(f'  Updating {target_info}')
+    target_info.write_text(json.dumps(info_data.to_dict(), cls=InfoJSONEncoder))
+
+    cli.log.info(f'{{fg_green}}Migration of keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}} complete!{{fg_reset}}')
+    cli.log.info(f"Verify build with {{fg_yellow}}qmk compile -kb {cli.args.keyboard} -km default{{fg_reset}}.")

lib/python/qmk/cli/multibuild.py

@@ -1,106 +0,0 @@
-"""Compile all keyboards.
-
-This will compile everything in parallel, for testing purposes.
-"""
-import os
-import re
-from pathlib import Path
-from subprocess import DEVNULL
-
-from milc import cli
-
-from qmk.constants import QMK_FIRMWARE
-from qmk.commands import _find_make, get_make_parallel_args
-import qmk.keyboard
-import qmk.keymap
-
-
-def _make_rules_mk_filter(key, value):
-    def _rules_mk_filter(keyboard_name):
-        rules_mk = qmk.keyboard.rules_mk(keyboard_name)
-        return True if key in rules_mk and rules_mk[key].lower() == str(value).lower() else False
-
-    return _rules_mk_filter
-
-
-def _is_split(keyboard_name):
-    rules_mk = qmk.keyboard.rules_mk(keyboard_name)
-    return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False
-
-
-@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
-@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
-@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
-@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.")
-@cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
-@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
-@cli.subcommand('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True)
-def multibuild(cli):
-    """Compile QMK Firmware against all keyboards.
-    """
-
-    make_cmd = _find_make()
-    if cli.args.clean:
-        cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL)
-
-    builddir = Path(QMK_FIRMWARE) / '.build'
-    makefile = builddir / 'parallel_kb_builds.mk'
-
-    keyboard_list = qmk.keyboard.list_keyboards()
-
-    filter_re = re.compile(r'^(?P<key>[A-Z0-9_]+)\s*=\s*(?P<value>[^#]+)$')
-    for filter_txt in cli.args.filter:
-        f = filter_re.match(filter_txt)
-        if f is not None:
-            keyboard_list = filter(_make_rules_mk_filter(f.group('key'), f.group('value')), keyboard_list)
-
-    keyboard_list = list(sorted(keyboard_list))
-
-    if len(keyboard_list) == 0:
-        return
-
-    builddir.mkdir(parents=True, exist_ok=True)
-    with open(makefile, "w") as f:
-        for keyboard_name in keyboard_list:
-            if qmk.keymap.locate_keymap(keyboard_name, cli.args.keymap) is not None:
-                keyboard_safe = keyboard_name.replace('/', '_')
-                # yapf: disable
-                f.write(
-                    f"""\
-all: {keyboard_safe}_binary
-{keyboard_safe}_binary:
-	@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}" || true
-	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{cli.args.keymap}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}"
-	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{cli.args.keymap}" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false {' '.join(cli.args.env)} \\
-		>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" 2>&1 \\
-		|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}"
-	@{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}" ; }} \\
-		|| {{ grep '\[WARNINGS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;33m[WARNINGS]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}" ; }} \\
-		|| printf "Build %-64s \e[1;32m[OK]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}"
-	@rm -f "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" || true
-"""# noqa
-                )
-                # yapf: enable
-
-                if cli.args.no_temp:
-                    # yapf: disable
-                    f.write(
-                        f"""\
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.elf" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.map" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.hex" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.bin" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.uf2" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}" || true
-	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{cli.args.keymap}" || true
-"""# noqa
-                    )
-                    # yapf: enable
-                f.write('\n')
-
-    cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
-
-    # Check for failures
-    failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]
-    if len(failures) > 0:
-        return False

lib/python/qmk/cli/new/keyboard.py

@@ -195,11 +195,6 @@ def new_keyboard(cli):
     cli.echo('')
 
     kb_name = cli.args.keyboard if cli.args.keyboard else prompt_keyboard()
-    user_name = cli.config.new_keyboard.name if cli.config.new_keyboard.name else prompt_user()
-    real_name = cli.args.realname or cli.config.new_keyboard.name if cli.args.realname or cli.config.new_keyboard.name else prompt_name(user_name)
-    default_layout = cli.args.layout if cli.args.layout else prompt_layout()
-    mcu = cli.args.type if cli.args.type else prompt_mcu()
-
     if not validate_keyboard_name(kb_name):
         cli.log.error('Keyboard names must contain only {fg_cyan}lowercase a-z{fg_reset}, {fg_cyan}0-9{fg_reset}, and {fg_cyan}_{fg_reset}! Please choose a different name.')
         return 1
@@ -208,6 +203,11 @@ def new_keyboard(cli):
         cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} already exists! Please choose a different name.')
         return 1
 
+    user_name = cli.config.new_keyboard.name if cli.config.new_keyboard.name else prompt_user()
+    real_name = cli.args.realname or cli.config.new_keyboard.name if cli.args.realname or cli.config.new_keyboard.name else prompt_name(user_name)
+    default_layout = cli.args.layout if cli.args.layout else prompt_layout()
+    mcu = cli.args.type if cli.args.type else prompt_mcu()
+
     # Preprocess any development_board presets
     if mcu in dev_boards:
         defaults_map = json_load(Path('data/mappings/defaults.hjson'))

lib/python/qmk/cli/new/keymap.py

@@ -1,12 +1,32 @@
 """This script automates the copying of the default keymap into your own keymap.
 """
 import shutil
-from pathlib import Path
 
-import qmk.path
+from milc import cli
+from milc.questions import question
+
+from qmk.path import is_keyboard, keymap
+from qmk.git import git_get_username
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from milc import cli
+
+
+def prompt_keyboard():
+    prompt = """{fg_yellow}Select Keyboard{style_reset_all}
+If you`re unsure you can view a full list of supported keyboards with {fg_yellow}qmk list-keyboards{style_reset_all}.
+
+Keyboard Name? """
+
+    return question(prompt)
+
+
+def prompt_user():
+    prompt = """
+{fg_yellow}Name Your Keymap{style_reset_all}
+Used for maintainer, copyright, etc
+
+Your GitHub Username? """
+    return question(prompt, default=git_get_username())
 
 
 @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Specify keyboard name. Example: 1upkeyboards/1up60hse')
@@ -17,32 +37,34 @@ from milc import cli
 def new_keymap(cli):
     """Creates a new keymap for the keyboard of your choosing.
     """
-    # ask for user input if keyboard or keymap was not provided in the command line
-    keyboard = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else input("Keyboard Name: ")
-    keymap = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else input("Keymap Name: ")
+    cli.log.info('{style_bright}Generating a new keymap{style_normal}')
+    cli.echo('')
 
-    # generate keymap paths
-    kb_path = Path('keyboards') / keyboard
-    keymap_path = qmk.path.keymap(keyboard)
-    keymap_path_default = keymap_path / 'default'
-    keymap_path_new = keymap_path / keymap
+    # ask for user input if keyboard or keymap was not provided in the command line
+    kb_name = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else prompt_keyboard()
+    user_name = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else prompt_user()
 
     # check directories
-    if not kb_path.exists():
-        cli.log.error('Keyboard %s does not exist!', kb_path)
+    if not is_keyboard(kb_name):
+        cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} does not exist! Please choose a valid name.')
         return False
 
+    # generate keymap paths
+    km_path = keymap(kb_name)
+    keymap_path_default = km_path / 'default'
+    keymap_path_new = km_path / user_name
+
     if not keymap_path_default.exists():
-        cli.log.error('Keyboard default %s does not exist!', keymap_path_default)
+        cli.log.error(f'Default keymap {{fg_cyan}}{keymap_path_default}{{fg_reset}} does not exist!')
         return False
 
     if keymap_path_new.exists():
-        cli.log.error('Keymap %s already exists!', keymap_path_new)
+        cli.log.error(f'Keymap {{fg_cyan}}{user_name}{{fg_reset}} already exists! Please choose a different name.')
         return False
 
     # create user directory with default keymap files
     shutil.copytree(keymap_path_default, keymap_path_new, symlinks=True)
 
     # end message to user
-    cli.log.info("%s keymap directory created in: %s", keymap, keymap_path_new)
-    cli.log.info("Compile a firmware with your new keymap by typing: \n\n\tqmk compile -kb %s -km %s\n", keyboard, keymap)
+    cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}')
+    cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.")

lib/python/qmk/cli/pyformat.py

@@ -1,24 +0,0 @@
-"""Point people to the new command name.
-"""
-import sys
-from pathlib import Path
-
-from milc import cli
-
-
-@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.")
-@cli.subcommand('Pointer to the new command name: qmk format-python.', hidden=False if cli.config.user.developer else True)
-def pyformat(cli):
-    """Pointer to the new command name: qmk format-python.
-    """
-    cli.log.warning('"qmk pyformat" has been renamed to "qmk format-python". Please use the new command in the future.')
-    argv = [sys.executable, *sys.argv]
-    argv[argv.index('pyformat')] = 'format-python'
-    script_path = Path(argv[1])
-    script_path_exe = Path(f'{argv[1]}.exe')
-
-    if not script_path.exists() and script_path_exe.exists():
-        # For reasons I don't understand ".exe" is stripped from the script name on windows.
-        argv[1] = str(script_path_exe)
-
-    return cli.run(argv, capture_output=False).returncode