Merge remote-tracking branch 'upstream/master' into fl16-july-merge

util/ci/discord-results.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import sys
+from pathlib import Path
+from discord_webhook import DiscordWebhook, DiscordEmbed
+
+parser = argparse.ArgumentParser(prog='discord-results.py', description='Sends a Discord webhook notification at the end of a CI run.')
+parser.add_argument('-b', '--branch')
+parser.add_argument('-k', '--keymap')
+parser.add_argument('-u', '--url')
+args = parser.parse_args()
+
+qmk_dir = Path(__file__).resolve().parents[2].resolve()
+
+keyboard_re = re.compile(r'CI Metadata: KEYBOARD=(.*)$', re.MULTILINE)
+keymap_re = re.compile(r'CI Metadata: KEYMAP=(.*)$', re.MULTILINE)
+
+successful_builds = sum([len(list(qmk_dir.glob(f'*.{extension}'))) for extension in ['uf2', 'bin', 'hex']])
+failures = list(sorted([f.resolve() for f in (qmk_dir / '.build/').glob('failed.log.*')]))
+failed_builds = []
+for f in failures:
+    with open(f) as fh:
+        data = fh.read()
+        kb = keyboard_re.search(data).group(1)
+        km = keymap_re.search(data).group(1)
+        failed_builds.append(f'{kb}:{km}')
+
+webhook = DiscordWebhook(url=os.getenv('DISCORD_WEBHOOK'), username="QMK GitHub CI")
+if len(failed_builds) > 0:
+    failstr = ''
+    for f in failed_builds:
+        if len(failstr) >= 1800:
+            failstr += '<<snip>>'
+            break
+        failstr += f'{f}\n'
+
+    embed = DiscordEmbed(title=f':infinity: CI Build Failure ({args.branch}, {args.keymap})', description=f'**{successful_builds}** builds succeeded, **{len(failed_builds)}** builds failed:```{failstr}```', color='ff9999')
+else:
+    embed = DiscordEmbed(title=f':infinity: CI Build Success ({args.branch}, {args.keymap})', description=f'**{successful_builds}** builds succeeded.', color='99ff99')
+
+embed.add_embed_field(name='Build Target', value=f'[**{args.branch}**](https://github.com/qmk/qmk_firmware/tree/{args.branch}) / **{args.keymap}** keymap')
+embed.add_embed_field(name='Workflow Run', value=f'[**Link**]({args.url})')
+embed.set_timestamp()
+
+webhook.add_embed(embed)
+webhook.execute()

util/ci/generate_failure_markdown.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+this_script="$(realpath "${BASH_SOURCE[0]}")"
+script_dir="$(realpath "$(dirname "$this_script")")"
+qmk_firmware_dir="$(realpath "$script_dir/../../")"
+
+dump_failure_info() {
+    local failure_file="$1"
+    local keyboard=$(cat "$failure_file" | grep 'CI Metadata: KEYBOARD=' | cut -d= -f2)
+    local keymap=$(cat "$failure_file" | grep 'CI Metadata: KEYMAP=' | cut -d= -f2)
+    echo "## ${keyboard}:${keymap}"
+    echo "\`\`\`"
+    cat "$failure_file" | sed -e $'s/\x1b\[[0-9;]*m//g' | grep -v "CI Metadata:" | grep -vP "(Entering|Leaving) directory"
+    echo "\`\`\`"
+}
+
+for failure_file in $(find "$qmk_firmware_dir/.build" -name 'failed.log.*' | sort); do
+    dump_failure_info "$failure_file"
+done
+
+exit 0

util/ci/requirements.txt

@@ -0,0 +1 @@
+discord-webhook

util/docker_build.sh

@@ -81,5 +81,5 @@ fi
 	-e ALT_GET_KEYBOARDS=true \
 	-e SKIP_GIT="$SKIP_GIT" \
 	-e MAKEFLAGS="$MAKEFLAGS" \
-	qmkfm/qmk_cli \
+	ghcr.io/qmk/qmk_cli \
 	make "$keyboard${keymap:+:$keymap}${target:+:$target}"

util/docker_cmd.sh

@@ -55,5 +55,5 @@ fi
 	$uid_arg \
 	-w /qmk_firmware \
 	-v "$dir":/qmk_firmware \
-	qmkfm/qmk_cli \
+	ghcr.io/qmk/qmk_cli \
 	"$@"

util/uf2conv.py

@@ -8,6 +8,7 @@ import os
 import os.path
 import argparse
 import json
+from time import sleep
 
 # Don't even need -b. hex has this embedded
 # > ./util/uf2conv.py .build/framework_ansi_default.hex -o ansi.uf2 -b 0x10000000 -f rp2040 --convert --blocks-reserved 1
@@ -171,10 +172,15 @@ class Block:
         flags = 0x0
         if familyid:
             flags |= 0x2000
+        if devicetype:
+            flags |= 0x8000
         hd = struct.pack("<IIIIIIII",
             UF2_MAGIC_START0, UF2_MAGIC_START1,
             flags, self.addr, 256, blockno + blocks_offset, blocks_offset + blocks_reserved + numblocks, familyid)
         hd += self.bytes[0:256]
+        if devicetype:
+            hd += bytearray(b'\x08\x29\xa7\xc8')
+            hd += bytearray(devicetype.to_bytes(4, 'little'))
         while len(hd) < 512 - 4:
             hd += b"\x00"
         hd += struct.pack("<I", UF2_MAGIC_END)
@@ -298,20 +304,10 @@ def main():
     parser = argparse.ArgumentParser(description='Convert to UF2 or flash directly.')
     parser.add_argument('input', metavar='INPUT', type=str, nargs='?',
                         help='input file (HEX, BIN or UF2)')
-    parser.add_argument('-b' , '--base', dest='base', type=str,
+    parser.add_argument('-b', '--base', dest='base', type=str,
                         default="0x2000",
                         help='set base address of application for BIN format (default: 0x2000)')
-    parser.add_argument('-o' , '--output', metavar="FILE", dest='output', type=str,
-                        help='write output to named file; defaults to "flash.uf2" or "flash.bin" where sensible')
-    parser.add_argument('-d' , '--device', dest="device_path",
-                        help='select a device path to flash')
-    parser.add_argument('-l' , '--list', action='store_true',
-                        help='list connected devices')
-    parser.add_argument('-c' , '--convert', action='store_true',
-                        help='do not flash, just convert')
-    parser.add_argument('-D' , '--deploy', action='store_true',
-                        help='just flash, do not convert')
-    parser.add_argument('-f' , '--family', dest='family', type=str,
+    parser.add_argument('-f', '--family', dest='family', type=str,
                         default="0x0",
                         help='specify familyID - number or name (default: 0x0)')
     parser.add_argument('--blocks-offset', dest='blocks_offset', type=str,
@@ -320,7 +316,21 @@ def main():
     parser.add_argument('--blocks-reserved', dest='blocks_reserved', type=str,
                         default="0x0",
                         help='TODO')
-    parser.add_argument('-C' , '--carray', action='store_true',
+    parser.add_argument('-t' , '--device-type', dest='devicetype', type=str,
+                        help='specify deviceTypeID extension tag - number')
+    parser.add_argument('-o', '--output', metavar="FILE", dest='output', type=str,
+                        help='write output to named file; defaults to "flash.uf2" or "flash.bin" where sensible')
+    parser.add_argument('-d', '--device', dest="device_path",
+                        help='select a device path to flash')
+    parser.add_argument('-l', '--list', action='store_true',
+                        help='list connected devices')
+    parser.add_argument('-c', '--convert', action='store_true',
+                        help='do not flash, just convert')
+    parser.add_argument('-D', '--deploy', action='store_true',
+                        help='just flash, do not convert')
+    parser.add_argument('-w', '--wait', action='store_true',
+                        help='wait for device to flash')
+    parser.add_argument('-C', '--carray', action='store_true',
                         help='convert binary file to a C array, not UF2')
     parser.add_argument('-i', '--info', action='store_true',
                         help='display header information from UF2, do not convert')
@@ -339,6 +349,9 @@ def main():
         except ValueError:
             error("Family ID needs to be a number or one of: " + ", ".join(families.keys()))
 
+    global devicetype
+    devicetype = int(args.devicetype, 0) if args.devicetype else None
+
     if args.list:
         list_drives()
     else:
@@ -368,21 +381,25 @@ def main():
             print("Converted to %s, output size: %d, start address: 0x%x" %
                   (ext, len(outbuf), appstartaddr))
         if args.convert or ext != "uf2":
-            drives = []
             if args.output == None:
                 args.output = "flash." + ext
-        else:
-            drives = get_drives()
-
         if args.output:
             write_file(args.output, outbuf)
-        else:
+        if ext == "uf2" and not args.convert and not args.info:
+            drives = get_drives()
             if len(drives) == 0:
                 error("No drive to deploy.")
-        if outbuf:
-            for d in drives:
-                print("Flashing %s (%s)" % (d, board_id(d)))
-                write_file(d + "/NEW.UF2", outbuf)
+                if args.wait:
+                    print("Waiting for drive to deploy...")
+                    while len(drives) == 0:
+                        sleep(0.1)
+                        drives = get_drives()
+                elif not args.output:
+                    error("No drive to deploy.")
+            if outbuf:
+                for d in drives:
+                    print("Flashing %s (%s)" % (d, board_id(d)))
+                    write_file(d + "/NEW.UF2", outbuf)
 
 
 if __name__ == "__main__":

util/uf2families.json

@@ -77,7 +77,7 @@
     {
         "id": "0x57755a57",
         "short_name": "STM32F4",
-        "description": "ST STM32F401"
+        "description": "ST STM32F4xx"
     },
     {
         "id": "0x5a18069b",
@@ -188,5 +188,20 @@
         "id": "0x9af03e33",
         "short_name": "GD32VF103",
         "description": "GigaDevice GD32VF103"
+    },
+    {
+        "id": "0x4f6ace52",
+        "short_name": "CSK4",
+        "description": "LISTENAI CSK300x/400x"
+    },
+    {
+        "id": "0x6e7348a8",
+        "short_name": "CSK6",
+        "description": "LISTENAI CSK60xx"
+    },
+    {
+        "id": "0x11de784a",
+        "short_name": "M0SENSE",
+        "description": "M0SENSE BL702"
     }
-]
+]

util/vagrant/Dockerfile

@@ -1,33 +0,0 @@
-FROM qmkfm/qmk_cli
-
-# Basic upgrades; install sudo and SSH.
-RUN apt-get update && apt-get install --no-install-recommends -y \
-        sudo \
-        openssh-server \
-    && rm -rf /var/lib/apt/lists/*
-RUN mkdir /var/run/sshd
-RUN sed -i 's/PermitRootLogin yes/PermitRootLogin no/' /etc/ssh/sshd_config
-RUN echo 'UseDNS no' >> /etc/ssh/sshd_config
-
-# Remove the policy file once we're finished installing software.
-# This allows invoke-rc.d and friends to work as expected.
-RUN rm /usr/sbin/policy-rc.d
-
-# Add the Vagrant user and necessary passwords.
-RUN groupadd vagrant
-RUN useradd -c "Vagrant" -g vagrant -d /home/vagrant -m -s /bin/bash vagrant
-RUN echo 'root:vagrant' | chpasswd
-RUN echo 'vagrant:vagrant' | chpasswd
-
-# Allow the vagrant user to use sudo without a password.
-RUN echo 'vagrant ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/vagrant
-
-# Install Vagrant's insecure public key so provisioning and 'vagrant ssh' work.
-RUN mkdir /home/vagrant/.ssh
-ADD https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub /home/vagrant/.ssh/authorized_keys
-RUN chmod 0600 /home/vagrant/.ssh/authorized_keys
-RUN chown -R vagrant:vagrant /home/vagrant/.ssh
-RUN chmod 0700 /home/vagrant/.ssh
-
-EXPOSE 22
-CMD ["/usr/sbin/sshd", "-D"]

util/vagrant/readme.md

@@ -1,12 +0,0 @@
-# QMK Vagrant Utilities
-
-## Dockerfile
-Vagrant-friendly `qmkfm/qmk_cli`.
-
-In order for the Docker provider and `vagrant ssh` to function the container has a few extra requirements.
-
-* vagrant user
-* ssh server
-  * configured with expected public key
-* sudo
-  * passwordless for vagrant user