Merge remote-tracking branch 'upstream/master' into fl16-july-merge

lib/python/qmk/c_parse.py

@@ -11,7 +11,7 @@ from milc import cli
 
 from qmk.comment_remover import comment_remover
 
-default_key_entry = {'x': -1, 'y': 0, 'w': 1}
+default_key_entry = {'x': -1, 'y': 0}
 single_comment_regex = re.compile(r'\s+/[/*].*$')
 multi_comment_regex = re.compile(r'/\*(.|\n)*?\*/', re.MULTILINE)
 layout_macro_define_regex = re.compile(r'^#\s*define')
@@ -90,8 +90,10 @@ def find_layouts(file):
                     cli.log.error('Invalid LAYOUT macro in %s: Empty parameter name in macro %s at pos %s.', file, macro_name, i)
                 elif key['label'] not in matrix_locations:
                     cli.log.error('Invalid LAYOUT macro in %s: Key %s in macro %s has no matrix position!', file, key['label'], macro_name)
+                elif len(matrix_locations.get(key['label'])) > 1:
+                    cli.log.error('Invalid LAYOUT macro in %s: Key %s in macro %s has multiple matrix positions (%s)', file, key['label'], macro_name, ', '.join(str(x) for x in matrix_locations[key['label']]))
                 else:
-                    key['matrix'] = matrix_locations[key['label']]
+                    key['matrix'] = matrix_locations[key['label']][0]
 
             parsed_layouts[macro_name] = {
                 'layout': parsed_layout,
@@ -186,7 +188,9 @@ def _parse_matrix_locations(matrix, file, macro_name):
         row = row.replace('{', '').replace('}', '')
         for col_num, identifier in enumerate(row.split(',')):
             if identifier != 'KC_NO':
-                matrix_locations[identifier] = [row_num, col_num]
+                if identifier not in matrix_locations:
+                    matrix_locations[identifier] = []
+                matrix_locations[identifier].append([row_num, col_num])
 
     return matrix_locations
 
@@ -213,10 +217,13 @@ def _coerce_led_token(_type, value):
         return value_map[value]
 
 
-def _validate_led_config(matrix, matrix_rows, matrix_indexes, position, position_raw, flags):
+def _validate_led_config(matrix, matrix_rows, matrix_cols, matrix_indexes, position, position_raw, flags):
     # TODO: Improve crude parsing/validation
     if len(matrix) != matrix_rows and len(matrix) != (matrix_rows / 2):
         raise ValueError("Unable to parse g_led_config matrix data")
+    for index, row in enumerate(matrix):
+        if len(row) != matrix_cols:
+            raise ValueError(f"Number of columns in row {index} ({len(row)}) does not match matrix ({matrix_cols})")
     if len(position) != len(flags):
         raise ValueError(f"Number of g_led_config physical positions ({len(position)}) does not match number of flags ({len(flags)})")
     if len(matrix_indexes) and (max(matrix_indexes) >= len(flags)):
@@ -230,32 +237,44 @@ def _validate_led_config(matrix, matrix_rows, matrix_indexes, position, position
 def _parse_led_config(file, matrix_cols, matrix_rows):
     """Return any 'raw' led/rgb matrix config
     """
-    matrix_raw = []
+    matrix = []
     position_raw = []
     flags = []
 
-    found_led_config = False
+    found_led_config_t = False
+    found_g_led_config = False
     bracket_count = 0
     section = 0
+    current_row_index = 0
+    current_row = []
+
     for _type, value in lex(_preprocess_c_file(file), CLexer()):
-        # Assume g_led_config..stuff..;
-        if value == 'g_led_config':
-            found_led_config = True
+        if not found_g_led_config:
+            # Check for type
+            if value == 'led_config_t':
+                found_led_config_t = True
+            # Type found, now check for name
+            elif found_led_config_t and value == 'g_led_config':
+                found_g_led_config = True
         elif value == ';':
-            found_led_config = False
-        elif found_led_config:
+            found_g_led_config = False
+        else:
             # Assume bracket count hints to section of config we are within
             if value == '{':
                 bracket_count += 1
                 if bracket_count == 2:
                     section += 1
             elif value == '}':
+                if section == 1 and bracket_count == 3:
+                    matrix.append(current_row)
+                    current_row = []
+                    current_row_index += 1
                 bracket_count -= 1
             else:
                 # Assume any non whitespace value here is important enough to stash
                 if _type in [Token.Literal.Number.Integer, Token.Literal.Number.Float, Token.Literal.Number.Hex, Token.Name]:
                     if section == 1 and bracket_count == 3:
-                        matrix_raw.append(_coerce_led_token(_type, value))
+                        current_row.append(_coerce_led_token(_type, value))
                     if section == 2 and bracket_count == 3:
                         position_raw.append(_coerce_led_token(_type, value))
                     if section == 3 and bracket_count == 2:
@@ -265,16 +284,15 @@ def _parse_led_config(file, matrix_cols, matrix_rows):
                     return None
 
     # Slightly better intrim format
-    matrix = list(_get_chunks(matrix_raw, matrix_cols))
     position = list(_get_chunks(position_raw, 2))
-    matrix_indexes = list(filter(lambda x: x is not None, matrix_raw))
+    matrix_indexes = list(filter(lambda x: x is not None, sum(matrix, [])))
 
     # If we have not found anything - bail with no error
     if not section:
         return None
 
     # Throw any validation errors
-    _validate_led_config(matrix, matrix_rows, matrix_indexes, position, position_raw, flags)
+    _validate_led_config(matrix, matrix_rows, matrix_cols, matrix_indexes, position, position_raw, flags)
 
     return (matrix, position, flags)
 

lib/python/qmk/cli/__init__.py

@@ -57,6 +57,7 @@ subcommands = [
     'qmk.cli.generate.keyboard_h',
     'qmk.cli.generate.keycodes',
     'qmk.cli.generate.keycodes_tests',
+    'qmk.cli.generate.make_dependencies',
     'qmk.cli.generate.rgb_breathe_table',
     'qmk.cli.generate.rules_mk',
     'qmk.cli.generate.version_h',

lib/python/qmk/cli/c2json.py

@@ -57,7 +57,7 @@ def c2json(cli):
         cli.args.output.parent.mkdir(parents=True, exist_ok=True)
         if cli.args.output.exists():
             cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak'))
-        cli.args.output.write_text(json.dumps(keymap_json, cls=InfoJSONEncoder))
+        cli.args.output.write_text(json.dumps(keymap_json, cls=InfoJSONEncoder, sort_keys=True))
 
         if not cli.args.quiet:
             cli.log.info('Wrote keymap to %s.', cli.args.output)

lib/python/qmk/cli/find.py

@@ -11,13 +11,21 @@ from qmk.search import search_keymap_targets
     action='append',
     default=[],
     help=  # noqa: `format-python` and `pytest` don't agree here.
-    "Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
+    "Filter the list of keyboards based on their info.json data. Accepts the formats key=value, function(key), or function(key,value), eg. 'features.rgblight=true'. Valid functions are 'absent', 'contains', 'exists' and 'length'. May be passed multiple times; all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
 )
+@cli.argument('-p', '--print', arg_only=True, action='append', default=[], help="For each matched target, print the value of the supplied info.json key. May be passed multiple times.")
 @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
 @cli.subcommand('Find builds which match supplied search criteria.')
 def find(cli):
     """Search through all keyboards and keymaps for a given search criteria.
     """
-    targets = search_keymap_targets(cli.args.keymap, cli.args.filter)
-    for target in targets:
-        print(f'{target[0]}:{target[1]}')
+
+    if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
+        cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
+
+    targets = search_keymap_targets(cli.args.keymap, cli.args.filter, cli.args.print)
+    for keyboard, keymap, print_vals in targets:
+        print(f'{keyboard}:{keymap}')
+
+        for key, val in print_vals:
+            print(f'    {key}={val}')

lib/python/qmk/cli/format/json.py

@@ -62,4 +62,4 @@ def format_json(cli):
                 json_file['layers'][layer_num] = current_layer
 
     # Display the results
-    print(json.dumps(json_file, cls=json_encoder))
+    print(json.dumps(json_file, cls=json_encoder, sort_keys=True))

lib/python/qmk/cli/format/python.py

@@ -7,7 +7,7 @@ from milc import cli
 from qmk.path import normpath
 
 py_file_suffixes = ('py',)
-py_dirs = ['lib/python']
+py_dirs = ['lib/python', 'util/ci']
 
 
 def yapf_run(files):

lib/python/qmk/cli/generate/api.py

@@ -8,7 +8,6 @@ from milc import cli
 
 from qmk.datetime import current_datetime
 from qmk.info import info_json
-from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
 from qmk.keymap import list_keymaps
 from qmk.keyboard import find_readme, list_keyboards
@@ -43,14 +42,14 @@ def _resolve_keycode_specs(output_folder):
         overall = load_spec(version)
 
         output_file = output_folder / f'constants/keycodes_{version}.json'
-        output_file.write_text(json.dumps(overall), encoding='utf-8')
+        output_file.write_text(json.dumps(overall, separators=(',', ':')), encoding='utf-8')
 
     for lang in list_languages():
         for version in list_versions(lang):
             overall = load_spec(version, lang)
 
             output_file = output_folder / f'constants/keycodes_{lang}_{version}.json'
-            output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
+            output_file.write_text(json.dumps(overall, separators=(',', ':')), encoding='utf-8')
 
     # Purge files consumed by 'load_spec'
     shutil.rmtree(output_folder / 'constants/keycodes/')
@@ -64,6 +63,12 @@ def _filtered_copy(src, dst):
         data = json_load(src)
 
         dst = dst.with_suffix('.json')
+        dst.write_text(json.dumps(data, separators=(',', ':')), encoding='utf-8')
+        return dst
+
+    if dst.suffix == '.jsonschema':
+        data = json_load(src)
+
         dst.write_text(json.dumps(data), encoding='utf-8')
         return dst
 
@@ -130,7 +135,7 @@ def generate_api(cli):
             }
 
         keyboard_dir.mkdir(parents=True, exist_ok=True)
-        keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_json}})
+        keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_json}}, separators=(',', ':'))
         if not cli.args.dry_run:
             keyboard_info.write_text(keyboard_json, encoding='utf-8')
             cli.log.debug('Wrote file %s', keyboard_info)
@@ -144,7 +149,7 @@ def generate_api(cli):
                 keymap_hjson = kb_json['keymaps'][keymap]['path']
                 keymap_json = v1_dir / keymap_hjson
                 keymap_json.parent.mkdir(parents=True, exist_ok=True)
-                keymap_json.write_text(json.dumps(json_load(Path(keymap_hjson))), encoding='utf-8')
+                keymap_json.write_text(json.dumps(json_load(Path(keymap_hjson)), separators=(',', ':')), encoding='utf-8')
                 cli.log.debug('Wrote keymap %s', keymap_json)
 
         if 'usb' in kb_json:
@@ -173,12 +178,12 @@ def generate_api(cli):
     _resolve_keycode_specs(v1_dir)
 
     # Write the global JSON files
-    keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'keyboards': kb_all}, cls=InfoJSONEncoder)
-    usb_json = json.dumps({'last_updated': current_datetime(), 'usb': usb_list}, cls=InfoJSONEncoder)
-    keyboard_list_json = json.dumps({'last_updated': current_datetime(), 'keyboards': keyboard_list}, cls=InfoJSONEncoder)
-    keyboard_aliases_json = json.dumps({'last_updated': current_datetime(), 'keyboard_aliases': keyboard_aliases}, cls=InfoJSONEncoder)
-    keyboard_metadata_json = json.dumps(keyboard_metadata, cls=InfoJSONEncoder)
-    constants_metadata_json = json.dumps({'last_updated': current_datetime(), 'constants': _list_constants(v1_dir)})
+    keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'keyboards': kb_all}, separators=(',', ':'))
+    usb_json = json.dumps({'last_updated': current_datetime(), 'usb': usb_list}, separators=(',', ':'))
+    keyboard_list_json = json.dumps({'last_updated': current_datetime(), 'keyboards': keyboard_list}, separators=(',', ':'))
+    keyboard_aliases_json = json.dumps({'last_updated': current_datetime(), 'keyboard_aliases': keyboard_aliases}, separators=(',', ':'))
+    keyboard_metadata_json = json.dumps(keyboard_metadata, separators=(',', ':'))
+    constants_metadata_json = json.dumps({'last_updated': current_datetime(), 'constants': _list_constants(v1_dir)}, separators=(',', ':'))
 
     if not cli.args.dry_run:
         keyboard_all_file.write_text(keyboard_all_json, encoding='utf-8')

lib/python/qmk/cli/generate/autocorrect_data.py

@@ -63,7 +63,13 @@ def parse_file(file_name: str) -> List[Tuple[str, str]]:
   """
 
     try:
+        import english_words
+        correct_words = english_words.get_english_words_set(['web2'], lower=True, alpha=True)
+    except AttributeError:
         from english_words import english_words_lower_alpha_set as correct_words
+        if not cli.args.quiet:
+            cli.echo('The english_words package is outdated, update by running:')
+            cli.echo('  {fg_cyan}python3 -m pip install english_words --upgrade')
     except ImportError:
         if not cli.args.quiet:
             cli.echo('Autocorrection will falsely trigger when a typo is a substring of a correctly spelled word.')

lib/python/qmk/cli/generate/compilation_database.py

@@ -15,6 +15,8 @@ from milc import cli, MILC
 from qmk.commands import create_make_command
 from qmk.constants import QMK_FIRMWARE
 from qmk.decorators import automagic_keyboard, automagic_keymap
+from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.keymap import keymap_completer
 
 
 @lru_cache(maxsize=10)
@@ -74,8 +76,8 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
     return records
 
 
-@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
-@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard\'s name')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap\'s name')
 @cli.subcommand('Create a compilation database.')
 @automagic_keyboard
 @automagic_keymap
@@ -104,7 +106,7 @@ def generate_compilation_database(cli: MILC) -> Union[bool, int]:
 
     if not command:
         cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
-        cli.echo('usage: qmk compiledb [-kb KEYBOARD] [-km KEYMAP]')
+        cli.echo('usage: qmk generate-compilation-database [-kb KEYBOARD] [-km KEYMAP]')
         return False
 
     # remove any environment variable overrides which could trip us up

lib/python/qmk/cli/generate/config_h.py

@@ -119,7 +119,9 @@ def generate_encoder_config(encoder_json, config_h_lines, postfix=''):
     config_h_lines.append(generate_define(f'ENCODERS_PAD_B{postfix}', f'{{ {", ".join(b_pads)} }}'))
 
     if None in resolutions:
-        cli.log.debug("Unable to generate ENCODER_RESOLUTION configuration")
+        cli.log.debug(f"Unable to generate ENCODER_RESOLUTION{postfix} configuration")
+    elif len(resolutions) == 0:
+        cli.log.debug(f"Skipping ENCODER_RESOLUTION{postfix} configuration")
     elif len(set(resolutions)) == 1:
         config_h_lines.append(generate_define(f'ENCODER_RESOLUTION{postfix}', resolutions[0]))
     else:

lib/python/qmk/cli/generate/info_json.py

@@ -76,7 +76,7 @@ def generate_info_json(cli):
     # Build the info.json file
     kb_info_json = info_json(cli.config.generate_info_json.keyboard)
     strip_info_json(kb_info_json)
-    info_json_text = json.dumps(kb_info_json, indent=4, cls=InfoJSONEncoder)
+    info_json_text = json.dumps(kb_info_json, indent=4, cls=InfoJSONEncoder, sort_keys=True)
 
     if cli.args.output:
         # Write to a file

lib/python/qmk/cli/generate/keyboard_h.py

@@ -11,12 +11,9 @@ from qmk.keyboard import keyboard_completer, keyboard_folder
 from qmk.constants import COL_LETTERS, ROW_LETTERS, GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
 
 
-def _generate_layouts(keyboard):
-    """Generates the layouts.h file.
+def _generate_layouts(keyboard, kb_info_json):
+    """Generates the layouts macros.
     """
-    # Build the info.json file
-    kb_info_json = info_json(keyboard)
-
     if 'matrix_size' not in kb_info_json:
         cli.log.error(f'{keyboard}: Invalid matrix config.')
         return []
@@ -65,6 +62,32 @@ def _generate_layouts(keyboard):
     return lines
 
 
+def _generate_keycodes(kb_info_json):
+    """Generates keyboard level keycodes.
+    """
+    if 'keycodes' not in kb_info_json:
+        return []
+
+    lines = []
+    lines.append('enum keyboard_keycodes {')
+
+    for index, item in enumerate(kb_info_json.get('keycodes')):
+        key = item["key"]
+        if index == 0:
+            lines.append(f'  {key} = QK_KB_0,')
+        else:
+            lines.append(f'  {key},')
+
+    lines.append('};')
+
+    for item in kb_info_json.get('keycodes', []):
+        key = item["key"]
+        for alias in item.get("aliases", []):
+            lines.append(f'#define {alias} {key}')
+
+    return lines
+
+
 @cli.argument('-i', '--include', nargs='?', arg_only=True, help='Optional file to include')
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@@ -73,8 +96,12 @@ def _generate_layouts(keyboard):
 def generate_keyboard_h(cli):
     """Generates the keyboard.h file.
     """
+    # Build the info.json file
+    kb_info_json = info_json(cli.args.keyboard)
+
     keyboard_h = cli.args.include
-    dd_layouts = _generate_layouts(cli.args.keyboard)
+    dd_layouts = _generate_layouts(cli.args.keyboard, kb_info_json)
+    dd_keycodes = _generate_keycodes(kb_info_json)
     valid_config = dd_layouts or keyboard_h
 
     # Build the layouts.h file.
@@ -87,6 +114,11 @@ def generate_keyboard_h(cli):
     if keyboard_h:
         keyboard_h_lines.append(f'#include "{Path(keyboard_h).name}"')
 
+    keyboard_h_lines.append('')
+    keyboard_h_lines.append('// Keycode content')
+    if dd_keycodes:
+        keyboard_h_lines.extend(dd_keycodes)
+
     # Protect against poorly configured keyboards
     if not valid_config:
         keyboard_h_lines.append('#error("<keyboard>.h is required unless your keyboard uses data-driven configuration. Please rename your keyboard\'s header file to <keyboard>.h")')

lib/python/qmk/cli/generate/keycodes.py

@@ -143,7 +143,7 @@ def generate_keycode_extras(cli):
     """
 
     # Build the header file.
-    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '#include "keymap.h"', '// clang-format off']
+    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '#include "keycodes.h"', '// clang-format off']
 
     keycodes = load_spec(cli.args.version, cli.args.lang)
 

lib/python/qmk/cli/generate/make_dependencies.py

@@ -0,0 +1,55 @@
+"""Used by the make system to generate dependency lists for each of the generated files.
+"""
+from pathlib import Path
+from milc import cli
+
+from argcomplete.completers import FilesCompleter
+
+from qmk.commands import dump_lines
+from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.keymap import keymap_completer, locate_keymap
+from qmk.path import normpath, FileType
+
+
+@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON.')
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate dependency file for.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.subcommand('Generates the list of dependencies associated with a keyboard build and its generated files.', hidden=True)
+def generate_make_dependencies(cli):
+    """Generates the list of dependent info.json, rules.mk, and config.h files for a keyboard.
+    """
+    interesting_files = [
+        'info.json',
+        'rules.mk',
+        'post_rules.mk',
+        'config.h',
+        'post_config.h',
+    ]
+
+    check_files = []
+
+    # Walk up the keyboard's directory tree looking for the files we're interested in
+    keyboards_root = Path('keyboards')
+    parent_path = Path('keyboards') / cli.args.keyboard
+    while parent_path != keyboards_root:
+        for file in interesting_files:
+            check_files.append(parent_path / file)
+        parent_path = parent_path.parent
+
+    # Find the keymap and include any of the interesting files
+    if cli.args.keymap is not None:
+        km = locate_keymap(cli.args.keyboard, cli.args.keymap)
+        if km is not None:
+            # keymap.json is only valid for the keymap, so check this one separately
+            check_files.append(km.parent / 'keymap.json')
+            # Add all the interesting files
+            for file in interesting_files:
+                check_files.append(km.parent / file)
+
+    # If we have a matching userspace, include those too
+    for file in interesting_files:
+        check_files.append(Path('users') / cli.args.keymap / file)
+
+    dump_lines(cli.args.output, [f'generated-files: $(wildcard {found})\n' for found in check_files])

lib/python/qmk/cli/git/submodule.py

@@ -7,14 +7,21 @@ from qmk import submodules
 
 REMOVE_DIRS = [
     'lib/ugfx',
-    'lib/pico-sdk',
     'lib/chibios-contrib/ext/mcux-sdk',
-    'lib/lvgl',
+]
+
+IGNORE_DIRS = [
+    'lib/arm_atsam',
+    'lib/fnv',
+    'lib/lib8tion',
+    'lib/python',
+    'lib/usbhost',
 ]
 
 
 @cli.argument('--check', arg_only=True, action='store_true', help='Check if the submodules are dirty, and display a warning if they are.')
 @cli.argument('--sync', arg_only=True, action='store_true', help='Shallow clone any missing submodules.')
+@cli.argument('-f', '--force', action='store_true', help='Flag to remove unexpected directories')
 @cli.subcommand('Git Submodule actions.')
 def git_submodule(cli):
     """Git Submodule actions
@@ -29,7 +36,15 @@ def git_submodule(cli):
                 cli.run(['git', 'submodule', 'update', '--depth=50', '--init', name], capture_output=False)
         return True
 
-    for folder in REMOVE_DIRS:
+    # can be the default behavior with: qmk config git_submodule.force=True
+    remove_dirs = REMOVE_DIRS
+    if cli.config.git_submodule.force:
+        # Also trash everything that isnt marked as "safe"
+        for path in normpath('lib').iterdir():
+            if not any(ignore in path.as_posix() for ignore in IGNORE_DIRS):
+                remove_dirs.append(path)
+
+    for folder in map(normpath, remove_dirs):
         if normpath(folder).is_dir():
             print(f"Removing '{folder}'")
             shutil.rmtree(folder)

lib/python/qmk/cli/info.py

@@ -18,6 +18,29 @@ from qmk.path import is_keyboard
 UNICODE_SUPPORT = sys.stdout.encoding.lower().startswith('utf')
 
 
+def _strip_api_content(info_json):
+    # Ideally this would only be added in the API pathway.
+    info_json.pop('platform', None)
+    info_json.pop('platform_key', None)
+    info_json.pop('processor_type', None)
+    info_json.pop('protocol', None)
+    info_json.pop('config_h_features', None)
+    info_json.pop('keymaps', None)
+    info_json.pop('keyboard_folder', None)
+    info_json.pop('parse_errors', None)
+    info_json.pop('parse_warnings', None)
+
+    for layout in info_json.get('layouts', {}).values():
+        layout.pop('filename', None)
+        layout.pop('c_macro', None)
+        layout.pop('json_layout', None)
+
+    if 'matrix_pins' in info_json:
+        info_json.pop('matrix_size', None)
+
+    return info_json
+
+
 def show_keymap(kb_info_json, title_caps=True):
     """Render the keymap in ascii art.
     """
@@ -81,7 +104,6 @@ def print_friendly_output(kb_info_json):
         cli.echo('{fg_blue}Maintainer{fg_reset}: QMK Community')
     else:
         cli.echo('{fg_blue}Maintainer{fg_reset}: %s', kb_info_json['maintainer'])
-    cli.echo('{fg_blue}Keyboard Folder{fg_reset}: %s', kb_info_json.get('keyboard_folder', 'Unknown'))
     cli.echo('{fg_blue}Layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys())))
     cli.echo('{fg_blue}Processor{fg_reset}: %s', kb_info_json.get('processor', 'Unknown'))
     cli.echo('{fg_blue}Bootloader{fg_reset}: %s', kb_info_json.get('bootloader', 'Unknown'))
@@ -141,6 +163,7 @@ def print_parsed_rules_mk(keyboard_name):
 @cli.argument('-f', '--format', default='friendly', arg_only=True, help='Format to display the data in (friendly, text, json) (Default: friendly).')
 @cli.argument('--ascii', action='store_true', default=not UNICODE_SUPPORT, help='Render layout box drawings in ASCII only.')
 @cli.argument('-r', '--rules-mk', action='store_true', help='Render the parsed values of the keyboard\'s rules.mk file.')
+@cli.argument('-a', '--api', action='store_true', help='Show fully processed info intended for API consumption.')
 @cli.subcommand('Keyboard information.')
 @automagic_keyboard
 @automagic_keymap
@@ -171,9 +194,12 @@ def info(cli):
     else:
         kb_info_json = info_json(cli.config.info.keyboard)
 
+    if not cli.args.api:
+        kb_info_json = _strip_api_content(kb_info_json)
+
     # Output in the requested format
     if cli.args.format == 'json':
-        print(json.dumps(kb_info_json, cls=InfoJSONEncoder))
+        print(json.dumps(kb_info_json, cls=InfoJSONEncoder, sort_keys=True))
         return True
     elif cli.args.format == 'text':
         print_dotted_output(kb_info_json)

lib/python/qmk/cli/json2c.py

@@ -5,7 +5,7 @@ from milc import cli
 
 import qmk.keymap
 import qmk.path
-from qmk.commands import parse_configurator_json
+from qmk.commands import dump_lines, parse_configurator_json
 
 
 @cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')
@@ -21,21 +21,8 @@ def json2c(cli):
     # Parse the configurator from json file (or stdin)
     user_keymap = parse_configurator_json(cli.args.filename)
 
-    # Environment processing
-    if cli.args.output and cli.args.output.name == '-':
-        cli.args.output = None
-
     # Generate the keymap
     keymap_c = qmk.keymap.generate_c(user_keymap)
 
-    if cli.args.output:
-        cli.args.output.parent.mkdir(parents=True, exist_ok=True)
-        if cli.args.output.exists():
-            cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak'))
-        cli.args.output.write_text(keymap_c)
-
-        if not cli.args.quiet:
-            cli.log.info('Wrote keymap to %s.', cli.args.output)
-
-    else:
-        print(keymap_c)
+    # Show the results
+    dump_lines(cli.args.output, keymap_c.split('\n'), cli.args.quiet)

lib/python/qmk/cli/mass_compile.py

@@ -9,9 +9,11 @@ from milc import cli
 
 from qmk.constants import QMK_FIRMWARE
 from qmk.commands import _find_make, get_make_parallel_args
+from qmk.keyboard import resolve_keyboard
 from qmk.search import search_keymap_targets
 
 
+@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap> to compile in parallel. Specifying this overrides all other target search options.")
 @cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@@ -37,7 +39,11 @@ def mass_compile(cli):
     builddir = Path(QMK_FIRMWARE) / '.build'
     makefile = builddir / 'parallel_kb_builds.mk'
 
-    targets = search_keymap_targets(cli.args.keymap, cli.args.filter)
+    if len(cli.args.builds) > 0:
+        targets = list(sorted(set([(resolve_keyboard(e[0]), e[1]) for e in [b.split(':') for b in cli.args.builds]])))
+    else:
+        targets = search_keymap_targets(cli.args.keymap, cli.args.filter)
+
     if len(targets) == 0:
         return
 
@@ -71,9 +77,6 @@ all: {keyboard_safe}_{keymap_name}_binary
                     f"""\
 	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.elf" 2>/dev/null || true
 	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.map" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.hex" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.bin" 2>/dev/null || true
-	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.uf2" 2>/dev/null || true
 	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}" || true
 	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{keymap_name}" || true
 """# noqa

lib/python/qmk/cli/migrate.py

@@ -75,7 +75,7 @@ def migrate(cli):
 
     # Finally write out updated info.json
     cli.log.info(f'  Updating {target_info}')
-    target_info.write_text(json.dumps(info_data.to_dict(), cls=InfoJSONEncoder))
+    target_info.write_text(json.dumps(info_data.to_dict(), cls=InfoJSONEncoder, sort_keys=True))
 
     cli.log.info(f'{{fg_green}}Migration of keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}} complete!{{fg_reset}}')
     cli.log.info(f"Verify build with {{fg_yellow}}qmk compile -kb {cli.args.keyboard} -km default{{fg_reset}}.")

lib/python/qmk/cli/new/keyboard.py

@@ -102,7 +102,7 @@ def augment_community_info(src, dest):
         item["matrix"] = [int(item["y"]), int(item["x"])]
 
     # finally write out the updated info.json
-    dest.write_text(json.dumps(info, cls=InfoJSONEncoder))
+    dest.write_text(json.dumps(info, cls=InfoJSONEncoder, sort_keys=True))
 
 
 def _question(*args, **kwargs):

lib/python/qmk/cli/new/keymap.py

@@ -5,7 +5,7 @@ import shutil
 from milc import cli
 from milc.questions import question
 
-from qmk.path import is_keyboard, keymap
+from qmk.path import is_keyboard, keymaps, keymap
 from qmk.git import git_get_username
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
@@ -50,9 +50,9 @@ def new_keymap(cli):
         return False
 
     # generate keymap paths
-    km_path = keymap(kb_name)
-    keymap_path_default = km_path / 'default'
-    keymap_path_new = km_path / user_name
+    keymaps_dirs = keymaps(kb_name)
+    keymap_path_default = keymap(kb_name, 'default')
+    keymap_path_new = keymaps_dirs[0] / user_name
 
     if not keymap_path_default.exists():
         cli.log.error(f'Default keymap {{fg_cyan}}{keymap_path_default}{{fg_reset}} does not exist!')

lib/python/qmk/cli/via2json.py

@@ -141,5 +141,5 @@ def via2json(cli):
     # Generate the keymap.json
     keymap_json = generate_json(cli.args.keymap, cli.args.keyboard, keymap_layout, keymap_data, macro_data)
 
-    keymap_lines = [json.dumps(keymap_json, cls=KeymapJSONEncoder)]
+    keymap_lines = [json.dumps(keymap_json, cls=KeymapJSONEncoder, sort_keys=True)]
     dump_lines(cli.args.output, keymap_lines, cli.args.quiet)

lib/python/qmk/commands.py

@@ -51,6 +51,9 @@ def create_make_target(target, dry_run=False, parallel=1, **env_vars):
     for key, value in env_vars.items():
         env.append(f'{key}={value}')
 
+    if cli.config.general.verbose:
+        env.append('VERBOSE=true')
+
     return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target]
 
 
@@ -175,9 +178,6 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=Fa
     if bootloader:
         make_command.append(bootloader)
 
-    for key, value in env_vars.items():
-        make_command.append(f'{key}={value}')
-
     make_command.extend([
         f'KEYBOARD={user_keymap["keyboard"]}',
         f'KEYMAP={user_keymap["keymap"]}',
@@ -198,6 +198,9 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=Fa
         'QMK_BIN="qmk"',
     ])
 
+    for key, value in env_vars.items():
+        make_command.append(f'{key}={value}')
+
     return make_command
 
 

lib/python/qmk/importers.py

@@ -5,7 +5,7 @@ import json
 
 from qmk.git import git_get_username
 from qmk.json_schema import validate
-from qmk.path import keyboard, keymap
+from qmk.path import keyboard, keymaps
 from qmk.constants import MCU2BOOTLOADER, LEGACY_KEYCODES
 from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder
 from qmk.json_schema import deep_update, json_load
@@ -84,14 +84,14 @@ def import_keymap(keymap_data):
     kb_name = keymap_data['keyboard']
     km_name = keymap_data['keymap']
 
-    km_folder = keymap(kb_name) / km_name
+    km_folder = keymaps(kb_name)[0] / km_name
     keyboard_keymap = km_folder / 'keymap.json'
 
     # This is the deepest folder in the expected tree
     keyboard_keymap.parent.mkdir(parents=True, exist_ok=True)
 
     # Dump out all those lovely files
-    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder))
+    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder, sort_keys=True))
 
     return (kb_name, km_name)
 
@@ -139,8 +139,8 @@ def import_keyboard(info_data, keymap_data=None):
     temp = json_load(keyboard_info)
     deep_update(temp, info_data)
 
-    keyboard_info.write_text(json.dumps(temp, cls=InfoJSONEncoder))
-    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder))
+    keyboard_info.write_text(json.dumps(temp, cls=InfoJSONEncoder, sort_keys=True))
+    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder, sort_keys=True))
 
     return kb_name
 

lib/python/qmk/info.py

@@ -50,18 +50,14 @@ def _valid_community_layout(layout):
     return (Path('layouts/default') / layout).exists()
 
 
-def _validate(keyboard, info_data):
-    """Perform various validation on the provided info.json data
+def _get_key_left_position(key):
+    # Special case for ISO enter
+    return key['x'] - 0.25 if key.get('h', 1) == 2 and key.get('w', 1) == 1.25 else key['x']
+
+
+def _additional_validation(keyboard, info_data):
+    """Non schema checks
     """
-    # First validate against the jsonschema
-    try:
-        validate(info_data, 'qmk.api.keyboard.v1')
-
-    except jsonschema.ValidationError as e:
-        json_path = '.'.join([str(p) for p in e.absolute_path])
-        cli.log.error('Invalid API data: %s: %s: %s', keyboard, json_path, e.message)
-        exit(1)
-
     layouts = info_data.get('layouts', {})
     layout_aliases = info_data.get('layout_aliases', {})
     community_layouts = info_data.get('community_layouts', [])
@@ -73,7 +69,7 @@ def _validate(keyboard, info_data):
 
     # Warn if physical positions are offset (at least one key should be at x=0, and at least one key at y=0)
     for layout_name, layout_data in layouts.items():
-        offset_x = min([k['x'] for k in layout_data['layout']])
+        offset_x = min([_get_key_left_position(k) for k in layout_data['layout']])
         if offset_x > 0:
             _log_warning(info_data, f'Layout "{layout_name}" is offset on X axis by {offset_x}')
 
@@ -103,6 +99,27 @@ def _validate(keyboard, info_data):
         if layout_name not in layouts and layout_name not in layout_aliases:
             _log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
 
+    # keycodes with length > 7 must have short forms for visualisation purposes
+    for decl in info_data.get('keycodes', []):
+        if len(decl["key"]) > 7:
+            if not decl.get("aliases", []):
+                _log_error(info_data, f'Keycode {decl["key"]} has no short form alias')
+
+
+def _validate(keyboard, info_data):
+    """Perform various validation on the provided info.json data
+    """
+    # First validate against the jsonschema
+    try:
+        validate(info_data, 'qmk.api.keyboard.v1')
+
+        _additional_validation(keyboard, info_data)
+
+    except jsonschema.ValidationError as e:
+        json_path = '.'.join([str(p) for p in e.absolute_path])
+        cli.log.error('Invalid API data: %s: %s: %s', keyboard, json_path, e.message)
+        exit(1)
+
 
 def info_json(keyboard):
     """Generate the info.json data for a specific keyboard.

lib/python/qmk/json_encoders.py

@@ -27,30 +27,56 @@ class QMKJSONEncoder(json.JSONEncoder):
 
         return float(obj)
 
-    def encode_list(self, obj):
+    def encode_dict(self, obj, path):
+        """Encode a dict-like object.
+        """
+        if obj:
+            self.indentation_level += 1
+
+            items = sorted(obj.items(), key=self.sort_dict) if self.sort_keys else obj.items()
+            output = [self.indent_str + f"{json.dumps(key)}: {self.encode(value, path + [key])}" for key, value in items]
+
+            self.indentation_level -= 1
+
+            return "{\n" + ",\n".join(output) + "\n" + self.indent_str + "}"
+        else:
+            return "{}"
+
+    def encode_dict_single_line(self, obj, path):
+        """Encode a dict-like object onto a single line.
+        """
+        return "{" + ", ".join(f"{json.dumps(key)}: {self.encode(value, path + [key])}" for key, value in sorted(obj.items(), key=self.sort_layout)) + "}"
+
+    def encode_list(self, obj, path):
         """Encode a list-like object.
         """
         if self.primitives_only(obj):
-            return "[" + ", ".join(self.encode(element) for element in obj) + "]"
+            return "[" + ", ".join(self.encode(value, path + [index]) for index, value in enumerate(obj)) + "]"
 
         else:
             self.indentation_level += 1
-            output = [self.indent_str + self.encode(element) for element in obj]
+
+            if path[-1] in ('layout', 'rotary'):
+                # These are part of a LED layout or encoder config, put them on a single line
+                output = [self.indent_str + self.encode_dict_single_line(value, path + [index]) for index, value in enumerate(obj)]
+            else:
+                output = [self.indent_str + self.encode(value, path + [index]) for index, value in enumerate(obj)]
+
             self.indentation_level -= 1
 
             return "[\n" + ",\n".join(output) + "\n" + self.indent_str + "]"
 
-    def encode(self, obj):
-        """Encode keymap.json objects for QMK.
+    def encode(self, obj, path=[]):
+        """Encode JSON objects for QMK.
         """
         if isinstance(obj, Decimal):
             return self.encode_decimal(obj)
 
         elif isinstance(obj, (list, tuple)):
-            return self.encode_list(obj)
+            return self.encode_list(obj, path)
 
         elif isinstance(obj, dict):
-            return self.encode_dict(obj)
+            return self.encode_dict(obj, path)
 
         else:
             return super().encode(obj)
@@ -71,30 +97,42 @@ class QMKJSONEncoder(json.JSONEncoder):
 class InfoJSONEncoder(QMKJSONEncoder):
     """Custom encoder to make info.json's a little nicer to work with.
     """
-    def encode_dict(self, obj):
-        """Encode info.json dictionaries.
+    def sort_layout(self, item):
+        """Sorts the hashes in a nice way.
         """
-        if obj:
-            if set(("x", "y")).issubset(obj.keys()):
-                # These are part of a layout/led_config, put them on a single line.
-                return "{ " + ", ".join(f"{self.encode(key)}: {self.encode(element)}" for key, element in sorted(obj.items())) + " }"
+        key = item[0]
 
-            else:
-                self.indentation_level += 1
-                output = [self.indent_str + f"{json.dumps(key)}: {self.encode(value)}" for key, value in sorted(obj.items(), key=self.sort_dict)]
-                self.indentation_level -= 1
-                return "{\n" + ",\n".join(output) + "\n" + self.indent_str + "}"
-        else:
-            return "{}"
+        if key == 'label':
+            return '00label'
 
-    def sort_dict(self, key):
+        elif key == 'matrix':
+            return '01matrix'
+
+        elif key == 'x':
+            return '02x'
+
+        elif key == 'y':
+            return '03y'
+
+        elif key == 'w':
+            return '04w'
+
+        elif key == 'h':
+            return '05h'
+
+        elif key == 'flags':
+            return '06flags'
+
+        return key
+
+    def sort_dict(self, item):
         """Forces layout to the back of the sort order.
         """
-        key = key[0]
+        key = item[0]
 
         if self.indentation_level == 1:
             if key == 'manufacturer':
-                return '10keyboard_name'
+                return '10manufacturer'
 
             elif key == 'keyboard_name':
                 return '11keyboard_name'
@@ -120,21 +158,7 @@ class InfoJSONEncoder(QMKJSONEncoder):
 class KeymapJSONEncoder(QMKJSONEncoder):
     """Custom encoder to make keymap.json's a little nicer to work with.
     """
-    def encode_dict(self, obj):
-        """Encode dictionary objects for keymap.json.
-        """
-        if obj:
-            self.indentation_level += 1
-            output_lines = [f"{self.indent_str}{json.dumps(key)}: {self.encode(value)}" for key, value in sorted(obj.items(), key=self.sort_dict)]
-            output = ',\n'.join(output_lines)
-            self.indentation_level -= 1
-
-            return f"{{\n{output}\n{self.indent_str}}}"
-
-        else:
-            return "{}"
-
-    def encode_list(self, obj):
+    def encode_list(self, obj, path):
         """Encode a list-like object.
         """
         if self.indentation_level == 2:
@@ -168,10 +192,10 @@ class KeymapJSONEncoder(QMKJSONEncoder):
 
             return "[\n" + ",\n".join(output) + "\n" + self.indent_str + "]"
 
-    def sort_dict(self, key):
+    def sort_dict(self, item):
         """Sorts the hashes in a nice way.
         """
-        key = key[0]
+        key = item[0]
 
         if self.indentation_level == 1:
             if key == 'version':

lib/python/qmk/keyboard.py

@@ -182,7 +182,7 @@ def render_layout(layout_data, render_ascii, key_labels=None):
 
         if x >= 0.25 and w == 1.25 and h == 2:
             render_key_isoenter(textpad, x, y, w, h, label, style)
-        elif w == 2.25 and h == 2:
+        elif w == 1.5 and h == 2:
             render_key_baenter(textpad, x, y, w, h, label, style)
         else:
             render_key_rect(textpad, x, y, w, h, label, style)
@@ -275,7 +275,7 @@ def render_key_baenter(textpad, x, y, w, h, label, style):
     w = ceil(w * 4)
     h = ceil(h * 3)
 
-    label_len = w - 2
+    label_len = w + 1
     label_leftover = label_len - len(label)
 
     if len(label) > label_len:
@@ -292,9 +292,9 @@ def render_key_baenter(textpad, x, y, w, h, label, style):
     lab_line = array('u', box_chars['v'] + label_middle + box_chars['v'])
     bot_line = array('u', box_chars['bl'] + label_border_bottom + box_chars['br'])
 
-    textpad[y][x + 3:x + w] = top_line
-    textpad[y + 1][x + 3:x + w] = mid_line
-    textpad[y + 2][x + 3:x + w] = mid_line
-    textpad[y + 3][x:x + w] = crn_line
-    textpad[y + 4][x:x + w] = lab_line
-    textpad[y + 5][x:x + w] = bot_line
+    textpad[y][x:x + w] = top_line
+    textpad[y + 1][x:x + w] = mid_line
+    textpad[y + 2][x:x + w] = mid_line
+    textpad[y + 3][x - 3:x + w] = crn_line
+    textpad[y + 4][x - 3:x + w] = lab_line
+    textpad[y + 5][x - 3:x + w] = bot_line

lib/python/qmk/keymap.py

@@ -25,13 +25,14 @@ __INCLUDES__
  * This file was generated by qmk json2c. You may or may not want to
  * edit it directly.
  */
+__KEYCODE_OUTPUT_GOES_HERE__
 
 const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
 __KEYMAP_GOES_HERE__
 };
 
 #if defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE)
-const uint16_t PROGMEM encoder_map[][NUM_ENCODERS][2] = {
+const uint16_t PROGMEM encoder_map[][NUM_ENCODERS][NUM_DIRECTIONS] = {
 __ENCODER_MAP_GOES_HERE__
 };
 #endif // defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE)
@@ -123,6 +124,29 @@ def _generate_macros_function(keymap_json):
     return macro_txt
 
 
+def _generate_keycodes_function(keymap_json):
+    """Generates keymap level keycodes.
+    """
+    lines = []
+    lines.append('enum keymap_keycodes {')
+
+    for index, item in enumerate(keymap_json.get('keycodes', [])):
+        key = item["key"]
+        if index == 0:
+            lines.append(f'  {key} = QK_USER_0,')
+        else:
+            lines.append(f'  {key},')
+
+    lines.append('};')
+
+    for item in keymap_json.get('keycodes', []):
+        key = item["key"]
+        for alias in item.get("aliases", []):
+            lines.append(f'#define {alias} {key}')
+
+    return lines
+
+
 def template_json(keyboard):
     """Returns a `keymap.json` template for a keyboard.
 
@@ -317,6 +341,12 @@ def generate_c(keymap_json):
         hostlang = f'#include "keymap_{keymap_json["host_language"]}.h"\n#include "sendstring_{keymap_json["host_language"]}.h"\n'
     new_keymap = new_keymap.replace('__INCLUDES__', hostlang)
 
+    keycodes = ''
+    if 'keycodes' in keymap_json and keymap_json['keycodes'] is not None:
+        keycodes_txt = _generate_keycodes_function(keymap_json)
+        keycodes = '\n'.join(keycodes_txt)
+    new_keymap = new_keymap.replace('__KEYCODE_OUTPUT_GOES_HERE__', keycodes)
+
     return new_keymap
 
 
@@ -349,7 +379,7 @@ def write_json(keyboard, keymap, layout, layers, macros=None):
     """
     keymap_json = generate_json(keyboard, keymap, layout, layers, macros=None)
     keymap_content = json.dumps(keymap_json)
-    keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.json'
+    keymap_file = qmk.path.keymaps(keyboard)[0] / keymap / 'keymap.json'
 
     return write_file(keymap_file, keymap_content)
 
@@ -376,7 +406,7 @@ def write(keymap_json):
             A list of macros for this keymap.
     """
     keymap_content = generate_c(keymap_json)
-    keymap_file = qmk.path.keymap(keymap_json['keyboard']) / keymap_json['keymap'] / 'keymap.c'
+    keymap_file = qmk.path.keymaps(keymap_json['keyboard'])[0] / keymap_json['keymap'] / 'keymap.c'
 
     return write_file(keymap_file, keymap_content)
 

lib/python/qmk/makefile.py

@@ -18,7 +18,7 @@ def parse_rules_mk_file(file, rules_mk=None):
 
     file = Path(file)
     if file.exists():
-        rules_mk_lines = file.read_text().split("\n")
+        rules_mk_lines = file.read_text(encoding='utf-8').split("\n")
 
         for line in rules_mk_lines:
             # Filter out comments

lib/python/qmk/painter.py

@@ -158,37 +158,45 @@ def convert_requested_format(im, format):
     ncolors = format["num_colors"]
     image_format = format["image_format"]
 
+    # -- Check if ncolors is valid
+    # Formats accepting several options
+    if image_format in ['IMAGE_FORMAT_GRAYSCALE', 'IMAGE_FORMAT_PALETTE']:
+        valid = [2, 4, 8, 16, 256]
+
+    # Formats expecting a particular number
+    else:
+        # Read number from specs dict, instead of hardcoding
+        for _, fmt in valid_formats.items():
+            if fmt["image_format"] == image_format:
+                # has to be an iterable, to use `in`
+                valid = [fmt["num_colors"]]
+                break
+
+    if ncolors not in valid:
+        raise ValueError(f"Number of colors must be: {', '.join(valid)}.")
+
     # Work out where we're getting the bytes from
     if image_format == 'IMAGE_FORMAT_GRAYSCALE':
-        # Ensure we have a valid number of colors for the palette
-        if ncolors <= 0 or ncolors > 256 or (ncolors & (ncolors - 1) != 0):
-            raise ValueError("Number of colors must be 2, 4, 16, or 256.")
         # If mono, convert input to grayscale, then to RGB, then grab the raw bytes corresponding to the intensity of the red channel
         im = ImageOps.grayscale(im)
         im = im.convert("RGB")
     elif image_format == 'IMAGE_FORMAT_PALETTE':
-        # Ensure we have a valid number of colors for the palette
-        if ncolors <= 0 or ncolors > 256 or (ncolors & (ncolors - 1) != 0):
-            raise ValueError("Number of colors must be 2, 4, 16, or 256.")
         # If color, convert input to RGB, palettize based on the supplied number of colors, then get the raw palette bytes
         im = im.convert("RGB")
         im = im.convert("P", palette=Image.ADAPTIVE, colors=ncolors)
-    elif image_format == 'IMAGE_FORMAT_RGB565':
-        # Ensure we have a valid number of colors for the palette
-        if ncolors != 65536:
-            raise ValueError("Number of colors must be 65536.")
-        # If color, convert input to RGB
-        im = im.convert("RGB")
-    elif image_format == 'IMAGE_FORMAT_RGB888':
-        # Ensure we have a valid number of colors for the palette
-        if ncolors != 1677216:
-            raise ValueError("Number of colors must be 16777216.")
-        # If color, convert input to RGB
+    elif image_format in ['IMAGE_FORMAT_RGB565', 'IMAGE_FORMAT_RGB888']:
+        # Convert input to RGB
         im = im.convert("RGB")
 
     return im
 
 
+def rgb_to565(r, g, b):
+    msb = ((r >> 3 & 0x1F) << 3) + (g >> 5 & 0x07)
+    lsb = ((g >> 2 & 0x07) << 5) + (b >> 3 & 0x1F)
+    return [msb, lsb]
+
+
 def convert_image_bytes(im, format):
     """Convert the supplied image to the equivalent bytes required by the QMK firmware.
     """
@@ -246,41 +254,25 @@ def convert_image_bytes(im, format):
 
     if image_format == 'IMAGE_FORMAT_RGB565':
         # Take the red, green, and blue channels
-        image_bytes_red = im.tobytes("raw", "R")
-        image_bytes_green = im.tobytes("raw", "G")
-        image_bytes_blue = im.tobytes("raw", "B")
-        image_pixels_len = len(image_bytes_red)
+        red = im.tobytes("raw", "R")
+        green = im.tobytes("raw", "G")
+        blue = im.tobytes("raw", "B")
 
         # No palette
         palette = None
 
-        bytearray = []
-        for x in range(image_pixels_len):
-            # 5 bits of red, 3 MSb of green
-            byte = ((image_bytes_red[x] >> 3 & 0x1F) << 3) + (image_bytes_green[x] >> 5 & 0x07)
-            bytearray.append(byte)
-            # 3 LSb of green, 5 bits of blue
-            byte = ((image_bytes_green[x] >> 2 & 0x07) << 5) + (image_bytes_blue[x] >> 3 & 0x1F)
-            bytearray.append(byte)
+        bytearray = [byte for r, g, b in zip(red, green, blue) for byte in rgb_to565(r, g, b)]
 
     if image_format == 'IMAGE_FORMAT_RGB888':
         # Take the red, green, and blue channels
-        image_bytes_red = im.tobytes("raw", "R")
-        image_bytes_green = im.tobytes("raw", "G")
-        image_bytes_blue = im.tobytes("raw", "B")
-        image_pixels_len = len(image_bytes_red)
+        red = im.tobytes("raw", "R")
+        green = im.tobytes("raw", "G")
+        blue = im.tobytes("raw", "B")
 
         # No palette
         palette = None
 
-        bytearray = []
-        for x in range(image_pixels_len):
-            byte = image_bytes_red[x]
-            bytearray.append(byte)
-            byte = image_bytes_green[x]
-            bytearray.append(byte)
-            byte = image_bytes_blue[x]
-            bytearray.append(byte)
+        bytearray = [byte for r, g, b in zip(red, green, blue) for byte in (r, g, b)]
 
     if len(bytearray) != expected_byte_count:
         raise Exception(f"Wrong byte count, was {len(bytearray)}, expected {expected_byte_count}")

lib/python/qmk/painter_qgf.py

@@ -372,8 +372,8 @@ def _save(im, fp, filename):
             delta_descriptor = QGFFrameDeltaDescriptorV1()
             delta_descriptor.left = location[0]
             delta_descriptor.top = location[1]
-            delta_descriptor.right = location[0] + size[0]
-            delta_descriptor.bottom = location[1] + size[1]
+            delta_descriptor.right = location[0] + size[0] - 1
+            delta_descriptor.bottom = location[1] + size[1] - 1
 
             # Write the delta frame to the output
             vprint(f'{f"Frame {idx:3d} delta":26s} {fp.tell():5d}d / {fp.tell():04X}h')

lib/python/qmk/path.py

@@ -36,8 +36,8 @@ def keyboard(keyboard_name):
     return Path('keyboards') / keyboard_name
 
 
-def keymap(keyboard_name):
-    """Locate the correct directory for storing a keymap.
+def keymaps(keyboard_name):
+    """Returns all of the `keymaps/` directories for a given keyboard.
 
     Args:
 
@@ -45,17 +45,36 @@ def keymap(keyboard_name):
             The name of the keyboard. Example: clueboard/66/rev3
     """
     keyboard_folder = keyboard(keyboard_name)
+    found_dirs = []
 
     for _ in range(MAX_KEYBOARD_SUBFOLDERS):
         if (keyboard_folder / 'keymaps').exists():
-            return (keyboard_folder / 'keymaps').resolve()
+            found_dirs.append((keyboard_folder / 'keymaps').resolve())
 
         keyboard_folder = keyboard_folder.parent
 
+    if len(found_dirs) > 0:
+        return found_dirs
+
     logging.error('Could not find the keymaps directory!')
     raise NoSuchKeyboardError('Could not find keymaps directory for: %s' % keyboard_name)
 
 
+def keymap(keyboard_name, keymap_name):
+    """Locate the directory of a given keymap.
+
+    Args:
+
+        keyboard_name
+            The name of the keyboard. Example: clueboard/66/rev3
+        keymap_name
+            The name of the keymap. Example: default
+    """
+    for keymap_dir in keymaps(keyboard_name):
+        if (keymap_dir / keymap_name).exists():
+            return (keymap_dir / keymap_name).resolve()
+
+
 def normpath(path):
     """Returns a `pathlib.Path()` object for a given path.
 

lib/python/qmk/search.py

@@ -45,7 +45,7 @@ def _load_keymap_info(keyboard, keymap):
         return (keyboard, keymap, keymap_json(keyboard, keymap))
 
 
-def search_keymap_targets(keymap='default', filters=[]):
+def search_keymap_targets(keymap='default', filters=[], print_vals=[]):
     targets = []
 
     with multiprocessing.Pool() as pool:
@@ -61,19 +61,48 @@ def search_keymap_targets(keymap='default', filters=[]):
             target_list = [(kb, keymap) for kb in filter(lambda kb: kb is not None, pool.starmap(_keymap_exists, [(kb, keymap) for kb in qmk.keyboard.list_keyboards()]))]
 
         if len(filters) == 0:
-            targets = target_list
+            targets = [(kb, km, {}) for kb, km in target_list]
         else:
             cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
             valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in pool.starmap(_load_keymap_info, target_list)]
 
+            function_re = re.compile(r'^(?P<function>[a-zA-Z]+)\((?P<key>[a-zA-Z0-9_\.]+)(,\s*(?P<value>[^#]+))?\)$')
             equals_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
-            exists_re = re.compile(r'^exists\((?P<key>[a-zA-Z0-9_\.]+)\)$')
-            for filter_txt in filters:
-                f = equals_re.match(filter_txt)
-                if f is not None:
-                    key = f.group('key')
-                    value = f.group('value')
-                    cli.log.info(f'Filtering on condition ("{key}" == "{value}")...')
+
+            for filter_expr in filters:
+                function_match = function_re.match(filter_expr)
+                equals_match = equals_re.match(filter_expr)
+
+                if function_match is not None:
+                    func_name = function_match.group('function').lower()
+                    key = function_match.group('key')
+                    value = function_match.group('value')
+
+                    if value is not None:
+                        if func_name == 'length':
+                            valid_keymaps = filter(lambda e, key=key, value=value: key in e[2] and len(e[2].get(key)) == int(value), valid_keymaps)
+                        elif func_name == 'contains':
+                            valid_keymaps = filter(lambda e, key=key, value=value: key in e[2] and value in e[2].get(key), valid_keymaps)
+                        else:
+                            cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}')
+                            continue
+
+                        cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}, {{fg_cyan}}{value}{{fg_reset}})...')
+                    else:
+                        if func_name == 'exists':
+                            valid_keymaps = filter(lambda e, key=key: key in e[2], valid_keymaps)
+                        elif func_name == 'absent':
+                            valid_keymaps = filter(lambda e, key=key: key not in e[2], valid_keymaps)
+                        else:
+                            cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}')
+                            continue
+
+                        cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}})...')
+
+                elif equals_match is not None:
+                    key = equals_match.group('key')
+                    value = equals_match.group('value')
+                    cli.log.info(f'Filtering on condition: {{fg_cyan}}{key}{{fg_reset}} == {{fg_cyan}}{value}{{fg_reset}}...')
 
                     def _make_filter(k, v):
                         expr = fnmatch.translate(v)
@@ -87,13 +116,10 @@ def search_keymap_targets(keymap='default', filters=[]):
                         return f
 
                     valid_keymaps = filter(_make_filter(key, value), valid_keymaps)
+                else:
+                    cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
+                    continue
 
-                f = exists_re.match(filter_txt)
-                if f is not None:
-                    key = f.group('key')
-                    cli.log.info(f'Filtering on condition (exists: "{key}")...')
-                    valid_keymaps = filter(lambda e: e[2].get(key) is not None, valid_keymaps)
-
-            targets = [(e[0], e[1]) for e in valid_keymaps]
+            targets = [(e[0], e[1], [(p, e[2].get(p)) for p in print_vals]) for e in valid_keymaps]
 
     return targets

lib/python/qmk/tests/minimal_info.json

@@ -4,7 +4,7 @@
     "layouts": {
         "LAYOUT": {
             "layout": [
-                { "label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0 }
+                {"label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0}
             ]
         }
     }

lib/python/qmk/tests/test_cli_commands.py

@@ -144,7 +144,7 @@ def test_list_keymaps_no_keyboard_found():
 def test_json2c():
     result = check_subcommand('json2c', 'keyboards/handwired/pytest/has_template/keymaps/default_json/keymap.json')
     check_returncode(result)
-    assert result.stdout == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT_ortho_1x1(KC_A)};\n\n'
+    assert result.stdout == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT_ortho_1x1(KC_A)};\n\n\n'
 
 
 def test_json2c_macros():
@@ -158,7 +158,7 @@ def test_json2c_macros():
 def test_json2c_stdin():
     result = check_subcommand_stdin('keyboards/handwired/pytest/has_template/keymaps/default_json/keymap.json', 'json2c', '-')
     check_returncode(result)
-    assert result.stdout == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT_ortho_1x1(KC_A)};\n\n'
+    assert result.stdout == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT_ortho_1x1(KC_A)};\n\n\n'
 
 
 def test_json2c_wrong_json():
@@ -168,7 +168,7 @@ def test_json2c_wrong_json():
 
 
 def test_json2c_no_json():
-    result = check_subcommand('json2c', 'keyboards/handwired/pytest/pytest.h')
+    result = check_subcommand('json2c', 'keyboards/handwired/pytest/basic/keymaps/default/keymap.c')
     check_returncode(result, [1])
     assert 'Invalid JSON encountered' in result.stdout
 
@@ -188,7 +188,11 @@ def test_info_keyboard_render():
     assert 'Keyboard Name: pytest' in result.stdout
     assert 'Processor: atmega32u4' in result.stdout
     assert 'Layouts:' in result.stdout
-    assert 'k0' in result.stdout
+
+    if is_windows:
+        assert '|  |' in result.stdout
+    else:
+        assert '│  │' in result.stdout
 
 
 def test_info_keymap_render():
@@ -291,7 +295,7 @@ def test_generate_version_h():
 def test_format_json_keyboard():
     result = check_subcommand('format-json', '--format', 'keyboard', 'lib/python/qmk/tests/minimal_info.json')
     check_returncode(result)
-    assert result.stdout == '{\n    "keyboard_name": "tester",\n    "maintainer": "qmk",\n    "layouts": {\n        "LAYOUT": {\n            "layout": [\n                { "label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0 }\n            ]\n        }\n    }\n}\n'
+    assert result.stdout == '{\n    "keyboard_name": "tester",\n    "maintainer": "qmk",\n    "layouts": {\n        "LAYOUT": {\n            "layout": [\n                {"label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0}\n            ]\n        }\n    }\n}\n'
 
 
 def test_format_json_keymap():
@@ -303,7 +307,7 @@ def test_format_json_keymap():
 def test_format_json_keyboard_auto():
     result = check_subcommand('format-json', '--format', 'auto', 'lib/python/qmk/tests/minimal_info.json')
     check_returncode(result)
-    assert result.stdout == '{\n    "keyboard_name": "tester",\n    "maintainer": "qmk",\n    "layouts": {\n        "LAYOUT": {\n            "layout": [\n                { "label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0 }\n            ]\n        }\n    }\n}\n'
+    assert result.stdout == '{\n    "keyboard_name": "tester",\n    "maintainer": "qmk",\n    "layouts": {\n        "LAYOUT": {\n            "layout": [\n                {"label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0}\n            ]\n        }\n    }\n}\n'
 
 
 def test_format_json_keymap_auto():

lib/python/qmk/tests/test_qmk_path.py

@@ -5,8 +5,8 @@ import qmk.path
 
 
 def test_keymap_pytest_basic():
-    path = qmk.path.keymap('handwired/pytest/basic')
-    assert path.samefile('keyboards/handwired/pytest/basic/keymaps')
+    path = qmk.path.keymap('handwired/pytest/basic', 'default')
+    assert path.samefile('keyboards/handwired/pytest/basic/keymaps/default')
 
 
 def test_normpath():