QMK CLI and JSON keymap support (qmk#6176)

* Script to generate keymap.c from JSON file.

* Support for keymap.json

* Add a warning about the keymap.c getting overwritten.

* Fix keymap generating

* Install the python deps

* Flesh out more of the python environment

* Remove defunct json2keymap

* Style everything with yapf

* Polish up python support

* Hide json keymap.c into the .build dir

* Polish up qmk-compile-json

* Make milc work with positional arguments

* Fix a couple small things

* Fix some errors and make the CLI more understandable

* Make the qmk wrapper more robust

* Add basic QMK Doctor

* Clean up docstrings and flesh them out as needed

* remove unused compile_firmware() function

.editorconfig

@@ -16,6 +16,10 @@ insert_final_newline = true
 trim_trailing_whitespace = false
 indent_size = 4
 
+[{qmk,*.py}]
+charset = utf-8
+max_line_length = 200
+
 # Make these match what we have in .gitattributes
 [*.mk]
 end_of_line = lf

.gitignore

@@ -70,3 +70,6 @@ util/Win_Check_Output.txt
 secrets.tar
 id_rsa_*
 /.vs
+
+# python things
+__pycache__

bin/qmk

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""CLI wrapper for running QMK commands.
+"""
+import os
+import subprocess
+import sys
+from glob import glob
+from time import strftime
+from importlib import import_module
+from importlib.util import find_spec
+
+# Add the QMK python libs to our path
+script_dir = os.path.dirname(os.path.realpath(__file__))
+qmk_dir = os.path.abspath(os.path.join(script_dir, '..'))
+python_lib_dir = os.path.abspath(os.path.join(qmk_dir, 'lib', 'python'))
+sys.path.append(python_lib_dir)
+
+# Change to the root of our checkout
+os.environ['ORIG_CWD'] = os.getcwd()
+os.chdir(qmk_dir)
+
+# Make sure our modules have been setup
+with open('requirements.txt', 'r') as fd:
+    for line in fd.readlines():
+        line = line.strip().replace('<', '=').replace('>', '=')
+
+        if line[0] == '#':
+            continue
+
+        if '#' in line:
+            line = line.split('#')[0]
+
+        module = line.split('=')[0] if '=' in line else line
+        if not find_spec(module):
+            print('Your QMK build environment is not fully setup!\n')
+            print('Please run `./util/qmk_install.sh` to setup QMK.')
+            exit(255)
+
+# Figure out our version
+command = ['git', 'describe', '--abbrev=6', '--dirty', '--always', '--tags']
+result = subprocess.run(command, text=True, capture_output=True)
+
+if result.returncode == 0:
+    os.environ['QMK_VERSION'] = 'QMK ' + result.stdout.strip()
+else:
+    os.environ['QMK_VERSION'] = 'QMK ' + strftime('%Y-%m-%d-%H:%M:%S')
+
+# Setup the CLI
+import milc
+milc.EMOJI_LOGLEVELS['INFO'] = '{fg_blue}ψ{style_reset_all}'
+
+# If we were invoked as `qmk <cmd>` massage sys.argv into `qmk-<cmd>`.
+# This means we can't accept arguments to the qmk script itself.
+script_name = os.path.basename(sys.argv[0])
+if script_name == 'qmk':
+    if len(sys.argv) == 1:
+        milc.cli.log.error('No subcommand specified!\n')
+
+    if len(sys.argv) == 1 or sys.argv[1] in ['-h', '--help']:
+        milc.cli.echo('usage: qmk <subcommand> [...]')
+        milc.cli.echo('\nsubcommands:')
+        subcommands = glob(os.path.join(qmk_dir, 'bin', 'qmk-*'))
+        for subcommand in sorted(subcommands):
+            subcommand = os.path.basename(subcommand).split('-', 1)[1]
+            milc.cli.echo('\t%s', subcommand)
+        milc.cli.echo('\nqmk <subcommand> --help for more information')
+        exit(1)
+
+    if sys.argv[1] in ['-V', '--version']:
+        milc.cli.echo(os.environ['QMK_VERSION'])
+        exit(0)
+
+    sys.argv[0] = script_name = '-'.join((script_name, sys.argv[1]))
+    del sys.argv[1]
+
+# Look for which module to import
+if script_name == 'qmk':
+    milc.cli.print_help()
+    exit(0)
+elif not script_name.startswith('qmk-'):
+    milc.cli.log.error('Invalid symlink, must start with "qmk-": %s', script_name)
+else:
+    subcommand = script_name.replace('-', '.').replace('_', '.').split('.')
+    subcommand.insert(1, 'cli')
+    subcommand = '.'.join(subcommand)
+
+    try:
+        import_module(subcommand)
+    except ModuleNotFoundError as e:
+        if e.__class__.__name__ != subcommand:
+            raise
+
+        milc.cli.log.error('Invalid subcommand! Could not import %s.', subcommand)
+        exit(1)
+
+if __name__ == '__main__':
+    milc.cli()

bin/qmk-compile-json

@@ -0,0 +1 @@
+qmk

bin/qmk-doctor

@@ -0,0 +1 @@
+qmk

bin/qmk-hello

@@ -0,0 +1 @@
+qmk

bin/qmk-json-keymap

@@ -0,0 +1 @@
+qmk

build_json.mk

@@ -0,0 +1,27 @@
+# Look for a json keymap file
+ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_5)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_4)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_3)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_2)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+endif
+
+# Generate the keymap.c
+ifneq ("$(KEYMAP_JSON)","")
+    _ = $(shell bin/qmk-json-keymap -f $(KEYMAP_JSON) -o $(KEYMAP_C))
+endif

build_keyboard.mk

@@ -98,31 +98,38 @@ MAIN_KEYMAP_PATH_3 := $(KEYBOARD_PATH_3)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_4 := $(KEYBOARD_PATH_4)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_5 := $(KEYBOARD_PATH_5)/keymaps/$(KEYMAP)
 
-ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_5)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_4)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_3)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_2)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_1)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
-else ifneq ($(LAYOUTS),)
-    include build_layout.mk
-else
-    $(error Could not find keymap)
-    # this state should never be reached
+# Check for keymap.json first, so we can regenerate keymap.c
+include build_json.mk
+
+ifeq ("$(wildcard $(KEYMAP_PATH))", "")
+    # Look through the possible keymap folders until we find a matching keymap.c
+    ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_5)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_4)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_3)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_2)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_1)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+    else ifneq ($(LAYOUTS),)
+        # If we haven't found a keymap yet fall back to community layouts
+        include build_layout.mk
+    else
+        $(error Could not find keymap)
+        # this state should never be reached
+    endif
 endif
 
 ifeq ($(strip $(CTPC)), yes)
@@ -313,7 +320,6 @@ ifneq ("$(wildcard $(USER_PATH)/config.h)","")
     CONFIG_H += $(USER_PATH)/config.h
 endif
 
-
 # Object files directory
 #     To put object files in current directory, use a dot (.), do NOT make
 #     this an empty or blank macro!
@@ -323,7 +329,7 @@ ifneq ("$(wildcard $(KEYMAP_PATH)/config.h)","")
     CONFIG_H += $(KEYMAP_PATH)/config.h
 endif
 
-# # project specific files
+# project specific files
 SRC += $(KEYBOARD_SRC) \
     $(KEYMAP_C) \
     $(QUANTUM_SRC)

docs/_summary.md

@@ -8,6 +8,7 @@
 
 * [QMK Basics](README.md)
   * [QMK Introduction](getting_started_introduction.md)
+  * [QMK CLI](cli.md)
   * [Contributing to QMK](contributing.md)
   * [How to Use Github](getting_started_github.md)
   * [Getting Help](getting_started_getting_help.md)
@@ -34,13 +35,16 @@
   * [Keyboard Guidelines](hardware_keyboard_guidelines.md)
   * [Config Options](config_options.md)
   * [Keycodes](keycodes.md)
+  * [Coding Conventions - C](coding_conventions_c.md)
+  * [Coding Conventions - Python](coding_conventions_python.md)
   * [Documentation Best Practices](documentation_best_practices.md)
   * [Documentation Templates](documentation_templates.md)
   * [Glossary](reference_glossary.md)
   * [Unit Testing](unit_testing.md)
   * [Useful Functions](ref_functions.md)
   * [Configurator Support](reference_configurator_support.md)
   * [info.json Format](reference_info_json.md)
+  * [Python Development](python_development.md)
 
 * [Features](features.md)
   * [Basic Keycodes](keycodes_basic.md)

docs/cli.md

@@ -0,0 +1,31 @@
+# QMK CLI
+
+This page describes how to setup and use the QMK CLI.
+
+# Overview
+
+The QMK CLI makes building and working with QMK keyboards easier. We have provided a number of commands to help you work with QMK:
+
+* `qmk compile-json`
+
+# Setup
+
+Simply add the `qmk_firmware/bin` directory to your `PATH`. You can run the `qmk` commands from any directory.
+
+```
+export PATH=$PATH:$HOME/qmk_firmware/bin
+```
+
+You may want to add this to your `.profile`, `.bash_profile`, `.zsh_profile`, or other shell startup scripts.
+
+# Commands
+
+## `qmk compile-json`
+
+This command allows you to compile JSON files you have downloaded from <https://config.qmk.fm>.
+
+**Usage**:
+
+```
+qmk compile-json mine.json
+```

docs/coding_conventions_c.md

@@ -0,0 +1,58 @@
+# Coding Conventions (C)
+
+Most of our style is pretty easy to pick up on, but right now it's not entirely consistent. You should match the style of the code surrounding your change, but if that code is inconsistent or unclear use the following guidelines:
+
+* We indent using four (4) spaces (soft tabs)
+* We use a modified One True Brace Style
+  * Opening Brace: At the end of the same line as the statement that opens the block
+  * Closing Brace: Lined up with the first character of the statement that opens the block
+  * Else If: Place the closing brace at the beginning of the line and the next opening brace at the end of the same line.
+  * Optional Braces: Always include optional braces.
+    * Good: if (condition) { return false; }
+    * Bad: if (condition) return false;
+* We encourage use of C style comments: `/* */`
+  * Think of them as a story describing the feature
+  * Use them liberally to explain why particular decisions were made.
+  * Do not write obvious comments
+  * If you not sure if a comment is obvious, go ahead and include it.
+* In general we don't wrap lines, they can be as long as needed. If you do choose to wrap lines please do not wrap any wider than 76 columns.
+* We use `#pragma once` at the start of header files rather than old-style include guards (`#ifndef THIS_FILE_H`, `#define THIS_FILE_H`, ..., `#endif`)
+* We accept both forms of preprocessor if's: `#ifdef DEFINED` and `#if defined(DEFINED)`
+  * If you are not sure which to prefer use the `#if defined(DEFINED)` form.
+  * Do not change existing code from one style to the other, except when moving to a multiple condition `#if`.
+  * Do not put whitespace between `#` and `if`.
+  * When deciding how (or if) to indent directives keep these points in mind:
+    * Readability is more important than consistency.
+    * Follow the file's existing style. If the file is mixed follow the style that makes sense for the section you are modifying.
+    * When choosing to indent you can follow the indention level of the surrounding C code, or preprocessor directives can have their own indent level. Choose the style that best communicates the intent of your code.
+
+Here is an example for easy reference:
+
+```c
+/* Enums for foo */
+enum foo_state {
+  FOO_BAR,
+  FOO_BAZ,
+};
+
+/* Returns a value */
+int foo(void) {
+  if (some_condition) {
+    return FOO_BAR;
+  } else {
+    return -1;
+  }
+}
+```
+
+# Auto-formatting with clang-format
+
+[Clang-format](https://clang.llvm.org/docs/ClangFormat.html) is part of LLVM and can automatically format your code for you, because ain't nobody got time to do it manually. We supply a configuration file for it that applies most of the coding conventions listed above. It will only change whitespace and newlines, so you will still have to remember to include optional braces yourself.
+
+Use the [full LLVM installer](http://llvm.org/builds/) to get clang-format on Windows, or use `sudo apt install clang-format` on Ubuntu.
+
+If you run it from the command-line, pass `-style=file` as an option and it will automatically find the .clang-format configuration file in the QMK root directory.
+
+If you use VSCode, the standard C/C++ plugin supports clang-format, alternatively there is a [separate extension](https://marketplace.visualstudio.com/items?itemName=LLVMExtensions.ClangFormat) for it.
+
+Some things (like LAYOUT macros) are destroyed by clang-format, so either don't run it on those files, or wrap the sensitive code in `// clang-format off` and `// clang-format on`.