first

rt-thread/tools/release/README.md

@@ -0,0 +1,52 @@
+# 版本发布前自动更新与部署
+
+在ENV环境下，并在release文件夹下执行 `python buildbot.py update` 可完成自动版本发布**前** **部分** 准备工作。 欢迎补充其他发布前自动化脚本。
+
+目前可以自动更新和部署的内容包括：
+
+1. 更新所有BSP工程，包括.config文件、rtconfig文件更新，以及Keil\IAR等工程的刷新
+2. STM32启动文件更新：
+   1. 对gcc的汇编启动文件中main替换为entry函数
+   2. 将启动文件heap降为0(Keil IAR)
+   3. 将GCC的堆大小扩展到0x400，与Keil IAR保持一致
+
+## clang-format代码自动格式化
+
+run-clang-format.py 根据`.clang-format`和`.clang-format-ignore`文件，使用clang-format工具对代码进行格式化。
+
+如果**不希望**对某个文件夹进行格式化，那么在该文件夹下增加一个`.clang-format`，内容为：
+
+```yaml
+---
+Language: Cpp
+DisableFormat: true
+---
+```
+
+如果**不希望**对某个代码片段进行格式化，那么在代码中插入`// clang-format off/on`：
+
+```c
+int formatted_code;
+// clang-format off
+    void    unformatted_code  ;
+// clang-format on
+void formatted_code_again;
+```
+
+使用以下命令，将对除了bsp、elmfat、lwip等文件夹之外的所有代码进行格式化：
+
+```shell
+# 安装clang-format
+pip install clang-format
+# 切换到RTT目录
+cd $RTT_ROOT
+# 执行格式化
+# -r递归子目录，-i是将格式化结果写入文件，-e是排除目录，-j是并行线程，.是当前目录
+python tools/release/run-clang-format.py -r -i -e bsp/**/* -j 10 .
+```
+
+如果格式化过程中提示以下错误，一般是文件中存在UTF-8编码无法识别的字符。
+
+```shell
+error: Command 'clang-format -i libcpu\aarch64\common\asm-fpu.h' returned non-zero exit status 1
+```

rt-thread/tools/release/buildbot.py

@@ -0,0 +1,88 @@
+import os
+import sys
+
+def usage():
+    print('%s all     -- build all bsp' % os.path.basename(sys.argv[0]))
+    print('%s clean   -- clean all bsp' % os.path.basename(sys.argv[0]))
+    print('%s update  -- update all prject files' % os.path.basename(sys.argv[0]))
+
+BSP_ROOT = os.path.join("..", "..", "bsp")
+
+if len(sys.argv) != 2:
+    usage()
+    sys.exit(0)
+
+def update_project_file(project_dir):
+    if os.path.isfile(os.path.join(project_dir, 'template.Uv2')):
+        print('prepare MDK3 project file on ' + project_dir)
+        command = ' --target=mdk -s'
+        os.system('scons --directory=' + project_dir + command + ' > 1.txt')
+
+    if os.path.isfile(os.path.join(project_dir, 'template.uvproj')):
+        print('prepare MDK4 project file on ' + project_dir)
+        command = ' --target=mdk4 -s'
+        os.system('scons --directory=' + project_dir + command + ' > 1.txt')
+
+    if os.path.isfile(os.path.join(project_dir, 'template.uvprojx')):
+        print('prepare MDK5 project file on ' + project_dir)
+        command = ' --target=mdk5 -s'
+        os.system('scons --directory=' + project_dir + command + ' > 1.txt')
+
+    if os.path.isfile(os.path.join(project_dir, 'template.ewp')):
+        print('prepare IAR project file on ' + project_dir)
+        command = ' --target=iar -s'
+        os.system('scons --directory=' + project_dir + command + ' > 1.txt')
+
+
+def update_all_project_files(root_path):
+    # current path is dir
+    if os.path.isdir(root_path):
+        projects = os.listdir(root_path)
+        # is a project path?
+        if "SConstruct" in projects:
+            try:
+                # update rtconfig.h and .config
+                if "Kconfig" in projects:
+                    if "win32" in sys.platform:
+                        retval = os.getcwd()
+                        os.chdir(root_path)
+                        os.system("menuconfig --silent")
+                        os.chdir(retval)
+                    else:
+                        os.system('scons --pyconfig-silent -C {0}'.format(root_path))
+                update_project_file(root_path)
+            except Exception as e:
+                print("error message: {}".format(e))
+                sys.exit(-1)
+        else:
+            for i in projects:
+                new_root_path = os.path.join(root_path, i)
+                update_all_project_files(new_root_path)
+
+# get command options
+command = ''
+if sys.argv[1] == 'all':
+    command = ' '
+elif sys.argv[1] == 'clean':
+    command = ' -c'
+elif sys.argv[1] == 'update':
+    print('begin to update all the bsp projects')
+
+    from stm32_update import stm32_update
+    stm32_update(os.path.join(BSP_ROOT, 'stm32'))
+
+    update_all_project_files(BSP_ROOT)
+
+    print('finished!')
+    sys.exit(0)
+else:
+    usage()
+    sys.exit(0)
+
+projects = os.listdir(BSP_ROOT)
+for item in projects:
+    project_dir = os.path.join(BSP_ROOT, item)
+    if os.path.isfile(os.path.join(project_dir, 'SConstruct')):
+        if os.system('scons --directory=' + project_dir + command) != 0:
+            print('build failed!!')
+            break

rt-thread/tools/release/run-clang-format.py

@@ -0,0 +1,426 @@
+#!/usr/bin/env python
+"""A wrapper script around clang-format, suitable for linting multiple files
+and to use for continuous integration.
+
+This is an alternative API for the clang-format command line.
+It runs over multiple files and directories in parallel.
+A diff output is produced and a sensible exit code is returned.
+
+"""
+
+from __future__ import print_function, unicode_literals
+
+import argparse
+import codecs
+import difflib
+import fnmatch
+import io
+import errno
+import multiprocessing
+import os
+import signal
+import subprocess
+import sys
+import traceback
+import platform
+
+from functools import partial
+
+try:
+    from subprocess import DEVNULL  # py3k
+except ImportError:
+    DEVNULL = open(os.devnull, "wb")
+
+
+DEFAULT_EXTENSIONS = "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx"
+DEFAULT_CLANG_FORMAT_IGNORE = ".clang-format-ignore"
+
+
+class ExitStatus:
+    SUCCESS = 0
+    DIFF = 1
+    TROUBLE = 2
+
+
+def excludes_from_file(ignore_file):
+    excludes = []
+    try:
+        with io.open(ignore_file, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.startswith("#"):
+                    # ignore comments
+                    continue
+                pattern = line.rstrip()
+                if not pattern:
+                    # allow empty lines
+                    continue
+                excludes.append(pattern)
+    except EnvironmentError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    return excludes
+
+
+def list_files(files, recursive=False, extensions=None, exclude=None):
+    if extensions is None:
+        extensions = []
+    if exclude is None:
+        exclude = []
+
+    out = []
+    for file in files:
+        if recursive and os.path.isdir(file):
+            for dirpath, dnames, fnames in os.walk(file):
+                fpaths = [
+                    os.path.relpath(os.path.join(dirpath, fname), os.getcwd())
+                    for fname in fnames
+                ]
+                for pattern in exclude:
+                    # os.walk() supports trimming down the dnames list
+                    # by modifying it in-place,
+                    # to avoid unnecessary directory listings.
+                    dnames[:] = [
+                        x
+                        for x in dnames
+                        if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)
+                    ]
+                    fpaths = [x for x in fpaths if not fnmatch.fnmatch(x, pattern)]
+
+                for f in fpaths:
+                    ext = os.path.splitext(f)[1][1:]
+                    if ext in extensions:
+                        out.append(f)
+        else:
+            out.append(file)
+    return out
+
+
+def make_diff(file, original, reformatted):
+    return list(
+        difflib.unified_diff(
+            original,
+            reformatted,
+            fromfile="{}\t(original)".format(file),
+            tofile="{}\t(reformatted)".format(file),
+            n=3,
+        )
+    )
+
+
+class DiffError(Exception):
+    def __init__(self, message, errs=None):
+        super(DiffError, self).__init__(message)
+        self.errs = errs or []
+
+
+class UnexpectedError(Exception):
+    def __init__(self, message, exc=None):
+        super(UnexpectedError, self).__init__(message)
+        self.formatted_traceback = traceback.format_exc()
+        self.exc = exc
+
+
+def run_clang_format_diff_wrapper(args, file):
+    try:
+        ret = run_clang_format_diff(args, file)
+        return ret
+    except DiffError:
+        raise
+    except Exception as e:
+        raise UnexpectedError("{}: {}: {}".format(file, e.__class__.__name__, e), e)
+
+
+def run_clang_format_diff(args, file):
+    # try:
+    #     with io.open(file, "r", encoding="utf-8") as f:
+    #         original = f.readlines()
+    # except IOError as exc:
+    #     raise DiffError(str(exc))
+
+    if args.in_place:
+        invocation = [args.clang_format_executable, "-i", file]
+    else:
+        invocation = [args.clang_format_executable, file]
+
+    if args.style:
+        invocation.extend(["--style", args.style])
+
+    if args.dry_run:
+        print(" ".join(invocation))
+        return [], []
+
+    # Use of utf-8 to decode the process output.
+    #
+    # Hopefully, this is the correct thing to do.
+    #
+    # It's done due to the following assumptions (which may be incorrect):
+    # - clang-format will returns the bytes read from the files as-is,
+    #   without conversion, and it is already assumed that the files use utf-8.
+    # - if the diagnostics were internationalized, they would use utf-8:
+    #   > Adding Translations to Clang
+    #   >
+    #   > Not possible yet!
+    #   > Diagnostic strings should be written in UTF-8,
+    #   > the client can translate to the relevant code page if needed.
+    #   > Each translation completely replaces the format string
+    #   > for the diagnostic.
+    #   > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation
+    #
+    # It's not pretty, due to Python 2 & 3 compatibility.
+    encoding_py3 = {}
+    if sys.version_info[0] >= 3:
+        encoding_py3["encoding"] = "utf-8"
+
+    try:
+        proc = subprocess.Popen(
+            invocation,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+            **encoding_py3
+        )
+    except OSError as exc:
+        raise DiffError(
+            "Command '{}' failed to start: {}".format(
+                subprocess.list2cmdline(invocation), exc
+            )
+        )
+    proc_stdout = proc.stdout
+    proc_stderr = proc.stderr
+    if sys.version_info[0] < 3:
+        # make the pipes compatible with Python 3,
+        # reading lines should output unicode
+        encoding = "utf-8"
+        proc_stdout = codecs.getreader(encoding)(proc_stdout)
+        proc_stderr = codecs.getreader(encoding)(proc_stderr)
+    # hopefully the stderr pipe won't get full and block the process
+    outs = list(proc_stdout.readlines())
+    errs = list(proc_stderr.readlines())
+    proc.wait()
+    if proc.returncode:
+        raise DiffError(
+            "Command '{}' returned non-zero exit status {}".format(
+                subprocess.list2cmdline(invocation), proc.returncode
+            ),
+            errs,
+        )
+    if args.in_place:
+        return [], errs
+    return make_diff(file, original, outs), errs
+
+
+def bold_red(s):
+    return "\x1b[1m\x1b[31m" + s + "\x1b[0m"
+
+
+def colorize(diff_lines):
+    def bold(s):
+        return "\x1b[1m" + s + "\x1b[0m"
+
+    def cyan(s):
+        return "\x1b[36m" + s + "\x1b[0m"
+
+    def green(s):
+        return "\x1b[32m" + s + "\x1b[0m"
+
+    def red(s):
+        return "\x1b[31m" + s + "\x1b[0m"
+
+    for line in diff_lines:
+        if line[:4] in ["--- ", "+++ "]:
+            yield bold(line)
+        elif line.startswith("@@ "):
+            yield cyan(line)
+        elif line.startswith("+"):
+            yield green(line)
+        elif line.startswith("-"):
+            yield red(line)
+        else:
+            yield line
+
+
+def print_diff(diff_lines, use_color):
+    if use_color:
+        diff_lines = colorize(diff_lines)
+    if sys.version_info[0] < 3:
+        sys.stdout.writelines((l.encode("utf-8") for l in diff_lines))
+    else:
+        sys.stdout.writelines(diff_lines)
+
+
+def print_trouble(prog, message, use_colors):
+    error_text = "error:"
+    if use_colors:
+        error_text = bold_red(error_text)
+    print("{}: {} {}".format(prog, error_text, message), file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--clang-format-executable",
+        metavar="EXECUTABLE",
+        help="path to the clang-format executable",
+        default="clang-format",
+    )
+    parser.add_argument(
+        "--extensions",
+        help="comma separated list of file extensions (default: {})".format(
+            DEFAULT_EXTENSIONS
+        ),
+        default=DEFAULT_EXTENSIONS,
+    )
+    parser.add_argument(
+        "-r",
+        "--recursive",
+        action="store_true",
+        help="run recursively over directories",
+    )
+    parser.add_argument(
+        "-d", "--dry-run", action="store_true", help="just print the list of files"
+    )
+    parser.add_argument(
+        "-i",
+        "--in-place",
+        action="store_true",
+        help="format file instead of printing differences",
+    )
+    parser.add_argument("files", metavar="file", nargs="+")
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        help="disable output, useful for the exit code",
+    )
+    parser.add_argument(
+        "-j",
+        metavar="N",
+        type=int,
+        default=0,
+        help="run N clang-format jobs in parallel" " (default number of cpus + 1)",
+    )
+    parser.add_argument(
+        "--color",
+        default="auto",
+        choices=["auto", "always", "never"],
+        help="show colored diff (default: auto)",
+    )
+    parser.add_argument(
+        "-e",
+        "--exclude",
+        metavar="PATTERN",
+        action="append",
+        default=[],
+        help="exclude paths matching the given glob-like pattern(s)"
+        " from recursive search",
+    )
+    parser.add_argument(
+        "--style",
+        default="file",
+        help="formatting style to apply (LLVM, Google, Chromium, Mozilla, WebKit)",
+    )
+
+    args = parser.parse_args()
+
+    # use default signal handling, like diff return SIGINT value on ^C
+    # https://bugs.python.org/issue14229#msg156446
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
+    try:
+        signal.SIGPIPE
+    except AttributeError:
+        # compatibility, SIGPIPE does not exist on Windows
+        pass
+    else:
+        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+    colored_stdout = False
+    colored_stderr = False
+    if args.color == "always":
+        colored_stdout = True
+        colored_stderr = True
+    elif args.color == "auto":
+        colored_stdout = sys.stdout.isatty()
+        colored_stderr = sys.stderr.isatty()
+
+    version_invocation = [args.clang_format_executable, str("--version")]
+    try:
+        subprocess.check_call(version_invocation, stdout=DEVNULL)
+    except subprocess.CalledProcessError as e:
+        print_trouble(parser.prog, str(e), use_colors=colored_stderr)
+        return ExitStatus.TROUBLE
+    except OSError as e:
+        print_trouble(
+            parser.prog,
+            "Command '{}' failed to start: {}".format(
+                subprocess.list2cmdline(version_invocation), e
+            ),
+            use_colors=colored_stderr,
+        )
+        return ExitStatus.TROUBLE
+
+    retcode = ExitStatus.SUCCESS
+
+    if os.path.exists(DEFAULT_CLANG_FORMAT_IGNORE):
+        excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE)
+    else:
+        excludes = []
+    excludes.extend(args.exclude)
+
+    files = list_files(
+        args.files,
+        recursive=args.recursive,
+        exclude=excludes,
+        extensions=args.extensions.split(","),
+    )
+
+    if not files:
+        return
+
+    njobs = args.j
+    if njobs == 0:
+        njobs = multiprocessing.cpu_count() + 1
+    njobs = min(len(files), njobs)
+
+    if njobs == 1:
+        # execute directly instead of in a pool,
+        # less overhead, simpler stacktraces
+        it = (run_clang_format_diff_wrapper(args, file) for file in files)
+        pool = None
+    else:
+        pool = multiprocessing.Pool(njobs)
+        it = pool.imap_unordered(partial(run_clang_format_diff_wrapper, args), files)
+        pool.close()
+    while True:
+        try:
+            outs, errs = next(it)
+        except StopIteration:
+            break
+        except DiffError as e:
+            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
+            retcode = ExitStatus.TROUBLE
+            sys.stderr.writelines(e.errs)
+        except UnexpectedError as e:
+            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
+            sys.stderr.write(e.formatted_traceback)
+            retcode = ExitStatus.TROUBLE
+            # stop at the first unexpected error,
+            # something could be very wrong,
+            # don't process all files unnecessarily
+            if pool:
+                pool.terminate()
+            break
+        else:
+            sys.stderr.writelines(errs)
+            if outs == []:
+                continue
+            if not args.quiet:
+                print_diff(outs, use_color=colored_stdout)
+            if retcode == ExitStatus.SUCCESS:
+                retcode = ExitStatus.DIFF
+    if pool:
+        pool.join()
+    return retcode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rt-thread/tools/release/stm32_update.py

@@ -0,0 +1,125 @@
+# Copyright (c) 2006-2022, RT-Thread Development Team
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Change Logs:
+# Date           Author       Notes
+# 2021-10-11     Meco Man     First version
+
+# STM32 startup assembly language file:
+# 1.replace main to entry (GCC)
+# 2.reduce the heap size as 0x000 (Keil IAR)
+# 3.extend the GCC stack size as 0x400, which is the same as Keil and IAR startup files.
+
+import os
+import re
+
+# replace 'bl main' to 'bl entry'
+def stm32update_main2entry(path):
+    oldline = ''
+    newline = ''
+
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            if os.path.splitext(file)[1] == '.s': # find .s files (Keil MDK)
+                file_path = os.path.join(root,file)
+                flag_need_replace = False
+                with open(file_path,'r+',) as f:
+                    while True:
+                        line = f.readline()
+                        if line == '':
+                            break
+                        elif ('bl' in line) and ('main' in line): # find 'bl main'
+                            oldline = line # bl main
+                            newline = line.replace('main', 'entry') # use 'entry' to replace 'main'
+                            flag_need_replace = True # mark that need to be replaced
+                            break
+
+                    if (flag_need_replace == True): # use 'entry' to replace 'main'
+                        f.seek(0)
+                        content = f.read()
+                        f.seek(0)
+                        f.truncate()
+                        newcontent = content.replace(oldline, newline)
+                        f.write(newcontent)
+
+#reduce the heap size as 0x000
+def stm32update_heap2zero(path):
+    oldline = ''
+    newline = ''
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            file_path = os.path.join(root,file)
+            if os.path.splitext(file)[1] == '.s': # find .s files (Keil MDK)
+                with open(file_path,'r+',) as f:
+                    flag_need_replace = False
+                    while True:
+                        line = f.readline()
+                        if line == '':
+                            break
+
+                        re_result = re.match('\s*Heap_Size\s+EQU\s+0[xX][0-9a-fA-F]+', line)
+                        if re_result != None:
+                            oldline = line
+                            newline = re.sub('0[xX][0-9a-fA-F]+','0x00000000', oldline)
+                            flag_need_replace = True
+                            break
+
+                    if flag_need_replace == True:
+                        f.seek(0)
+                        content = f.read()
+                        f.seek(0)
+                        f.truncate()
+                        newcontent = content.replace(oldline, newline)
+                        f.write(newcontent)
+
+            elif os.path.splitext(file)[1] == '.icf': # find .icf files (IAR)
+                with open(file_path,'r+',) as f:
+                    flag_need_replace = False
+                    while True:
+                        line = f.readline()
+                        if line == '':
+                            break
+
+                        re_result = re.match('\s*define\s+symbol\s+__ICFEDIT_size_heap__\s*=\s*0[xX][0-9a-fA-F]+', line)
+                        if re_result != None:
+                            oldline = line
+                            newline = re.sub('0[xX][0-9a-fA-F]+','0x000', oldline)
+                            flag_need_replace = True
+                            break
+
+                    if flag_need_replace == True:
+                        f.seek(0)
+                        content = f.read()
+                        f.seek(0)
+                        f.truncate()
+                        newcontent = content.replace(oldline, newline)
+                        f.write(newcontent)
+
+            elif os.path.splitext(file)[1] == '.lds': # find .lds files (GCC)
+                with open(file_path,'r+',) as f:
+                    flag_need_replace = False
+                    while True:
+                        line = f.readline()
+                        if line == '':
+                            break
+
+                        re_result = re.match('\s*_system_stack_size\s*=\s*0[xX][0-9a-fA-F]+', line)
+                        if re_result != None:
+                            oldline = line
+                            newline = re.sub('0[xX][0-9a-fA-F]+','0x400', oldline)
+                            flag_need_replace = True
+                            break
+
+                    if flag_need_replace == True:
+                        f.seek(0)
+                        content = f.read()
+                        f.seek(0)
+                        f.truncate()
+                        newcontent = content.replace(oldline, newline)
+                        f.write(newcontent)
+
+
+def stm32_update(path):
+    stm32update_main2entry(path)
+    stm32update_heap2zero(path)