james/SumProject
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

first

Browse Source
This commit is contained in:
james
2024-08-05 20:57:09 +08:00
commit 46d9ee7795
3020 changed files with 1725767 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
rt-thread/tools/ci/bsp_buildings.py Normal file
View File

@@ -0,0 +1,183 @@
import os
import shutil
import re
import multiprocessing
def add_summary(text):
"""
add summary to github action.
"""
os.system(f'echo "{text}" >> $GITHUB_STEP_SUMMARY ;')
def run_cmd(cmd, output_info=True):
"""
run command and return output and result.
"""
print('\033[1;32m' + cmd + '\033[0m')
output_str_list = []
res = 0
if output_info:
res = os.system(cmd + " > output.txt 2>&1")
else:
res = os.system(cmd + " > /dev/null 2>output.txt")
with open("output.txt", "r") as file:
output_str_list = file.readlines()
for line in output_str_list:
print(line, end='')
os.remove("output.txt")
return output_str_list, res
def build_bsp(bsp, scons_args=''):
"""
build bsp.
cd {rtt_root}
scons -C bsp/{bsp} --pyconfig-silent > /dev/null
cd {rtt_root}/bsp/{bsp}
pkgs --update > /dev/null
pkgs --list
cd {rtt_root}
scons -C bsp/{bsp} -j{nproc} {scons_args}
cd {rtt_root}/bsp/{bsp}
scons -c > /dev/null
rm -rf packages
"""
success = True
os.chdir(rtt_root)
if os.path.exists(f"{rtt_root}/bsp/{bsp}/Kconfig"):
os.chdir(rtt_root)
run_cmd(f'scons -C bsp/{bsp} --pyconfig-silent', output_info=False)
os.chdir(f'{rtt_root}/bsp/{bsp}')
run_cmd('pkgs --update', output_info=False)
run_cmd('pkgs --list')
nproc = multiprocessing.cpu_count()
os.chdir(rtt_root)
cmd = f'scons -C bsp/{bsp} -j{nproc} {scons_args}'
__, res = run_cmd(cmd, output_info=False)
if res != 0:
success = False
os.chdir(f'{rtt_root}/bsp/{bsp}')
run_cmd('scons -c', output_info=False)
pkg_dir = os.path.join(rtt_root, 'bsp', bsp, 'packages')
shutil.rmtree(pkg_dir, ignore_errors=True)
return success
def append_file(source_file, destination_file):
"""
append file to another file.
"""
with open(source_file, 'r') as source:
with open(destination_file, 'a') as destination:
for line in source:
destination.write(line)
def check_scons_args(file_path):
args = []
with open(file_path, 'r') as file:
for line in file:
match = re.search(r'#\s*scons:\s*(.*)', line)
if match:
args.append(match.group(1).strip())
return ' '.join(args)
def build_bsp_attachconfig(bsp, attach_file):
"""
build bsp with attach config.
cp bsp/{bsp}/.config bsp/{bsp}/.config.origin
cat .ci/attachconfig/{attach_file} >> bsp/{bsp}/.config
build_bsp()
cp bsp/{bsp}/.config.origin bsp/{bsp}/.config
rm bsp/{bsp}/.config.origin
"""
config_file = os.path.join(rtt_root, 'bsp', bsp, '.config')
config_bacakup = config_file+'.origin'
shutil.copyfile(config_file, config_bacakup)
attachconfig_dir = os.path.join(rtt_root, 'bsp', bsp, '.ci/attachconfig')
attach_path = os.path.join(attachconfig_dir, attach_file)
append_file(attach_path, config_file)
scons_args = check_scons_args(attach_path)
res = build_bsp(bsp, scons_args)
shutil.copyfile(config_bacakup, config_file)
os.remove(config_bacakup)
return res
if __name__ == "__main__":
"""
build all bsp and attach config.
1. build all bsp.
2. build all bsp with attach config.
"""
failed = 0
count = 0
rtt_root = os.getcwd()
srtt_bsp = os.getenv('SRTT_BSP').split(',')
for bsp in srtt_bsp:
count += 1
print(f"::group::Compiling BSP: =={count}=== {bsp} ====")
res = build_bsp(bsp)
if not res:
print(f"::error::build {bsp} failed")
add_summary(f"- ❌ build {bsp} failed.")
failed += 1
else:
add_summary(f'- ✅ build {bsp} success.')
print("::endgroup::")
attach_dir = os.path.join(rtt_root, 'bsp', bsp, '.ci/attachconfig')
attach_list = []
for root, dirs, files in os.walk(attach_dir):
for file in files:
file_path = os.path.join(root, file)
relative_path = os.path.relpath(file_path, attach_dir)
attach_list.append(relative_path)
for attach_file in attach_list:
count += 1
print(f"::group::\tCompiling BSP: =={count}=== {bsp} {attach_file}===")
res = build_bsp_attachconfig(bsp, attach_file)
if not res:
print(f"::error::build {bsp} {attach_file} failed.")
add_summary(f'\t- ❌ build {attach_file} failed.')
failed += 1
else:
add_summary(f'\t- ✅ build {attach_file} success.')
print("::endgroup::")
exit(failed)

98
rt-thread/tools/ci/compile_bsp_with_drivers.py Normal file
View File

@@ -0,0 +1,98 @@
#
# Copyright (c) 2006-2023, RT-Thread Development Team
#
# SPDX-License-Identifier: Apache-2.0
#
# Change Logs:
# Date Author Notes
# 2023-06-27 dejavudwh the first version
#
import subprocess
import logging
import os
CONFIG_BSP_USING_X = ["CONFIG_BSP_USING_UART", "CONFIG_BSP_USING_I2C", "CONFIG_BSP_USING_SPI", "CONFIG_BSP_USING_ADC", "CONFIG_BSP_USING_DAC"]
def init_logger():
log_format = "[%(filename)s %(lineno)d %(levelname)s] %(message)s "
date_format = '%Y-%m-%d %H:%M:%S %a '
logging.basicConfig(level=logging.INFO,
format=log_format,
datefmt=date_format,
)
def diff():
result = subprocess.run(['git', 'diff', '--name-only', 'HEAD', 'origin/master', '--diff-filter=ACMR', '--no-renames', '--full-index'], stdout = subprocess.PIPE)
file_list = result.stdout.decode().strip().split('\n')
logging.info(file_list)
bsp_paths = set()
for file in file_list:
if "bsp/" in file:
logging.info("Modifed file: {}".format(file))
bsp_paths.add(file)
dirs = set()
for dir in bsp_paths:
dir = os.path.dirname(dir)
while "bsp/" in dir:
files = os.listdir(dir)
if ".config" in files and "rt-thread.elf" not in files and not dir.endswith("bsp"):
logging.info("Found bsp path: {}".format(dir))
dirs.add(dir)
break
new_dir = os.path.dirname(dir)
dir = new_dir
return dirs
def check_config_in_line(line):
for config in CONFIG_BSP_USING_X:
if config in line and '#' in line:
logging.info("Found in {}".format(line))
return config
return ""
def check_config_in_file(file_path):
configs = set()
found = False
try:
with open(file_path, 'r') as file:
for line in file:
line.strip()
if found:
res = check_config_in_line(line)
if res:
configs.add(res)
elif "On-chip Peripheral Drivers" in line:
logging.info("Found On-chip Peripheral Drivers")
found = True
except FileNotFoundError:
logging.error("The .config file does not exist for this BSP, please recheck the file directory!")
return configs
def modify_config(file_path, configs):
with open(file_path + "/rtconfig.h", 'a') as file:
for item in configs:
define1 = item.replace("CONFIG_BSP", "BSP")
define2 = item.replace("CONFIG_BSP", "RT")
file.write("#define " + define1 + "\n")
file.write("#define " + define2 + "\n")
def recompile_bsp(dir):
logging.info("recomplie bsp: {}".format(dir))
os.system("scons -C " + dir)
if __name__ == '__main__':
init_logger()
recompile_bsp_dirs = diff()
for dir in recompile_bsp_dirs:
dot_config_path = dir + "/" + ".config"
configs = check_config_in_file(dot_config_path)
logging.info("add config:")
logging.info(configs)
logging.info("Add configurations and recompile!")
modify_config(dir, configs)
recompile_bsp(dir)

81
rt-thread/tools/ci/cpp_check.py Normal file
View File

@@ -0,0 +1,81 @@
#
# Copyright (c) 2006-2023, RT-Thread Development Team
#
# SPDX-License-Identifier: Apache-2.0
#
# Change Logs:
# Date Author Notes
# 2023-05-16 dejavudwh the first version
#
import click
import logging
import subprocess
import sys
import format_ignore
class CPPCheck:
def __init__(self, file_list):
self.file_list = file_list
def check(self):
file_list_filtered = [file for file in self.file_list if file.endswith(('.c', '.cpp', '.cc', '.cxx'))]
logging.info("Start to static code analysis.")
check_result = True
for file in file_list_filtered:
result = subprocess.run(
[
'cppcheck',
'-DRT_ASSERT(x)=',
'-DRTM_EXPORT(x)=',
'-Drt_list_for_each_entry(a,b,c)=a=(void*)b;',
'-I include',
'-I thread/components/finsh',
# it's okay because CI will do the real compilation to check this
'--suppress=syntaxError',
'--enable=warning',
'performance',
'portability',
'--inline-suppr',
'--error-exitcode=1',
'--force',
file
],
stdout = subprocess.PIPE, stderr = subprocess.PIPE)
logging.info(result.stdout.decode())
logging.info(result.stderr.decode())
if result.stderr:
check_result = False
return check_result
@click.group()
@click.pass_context
def cli(ctx):
pass
@cli.command()
def check():
"""
static code analysis(cppcheck).
"""
format_ignore.init_logger()
# get modified files list
checkout = format_ignore.CheckOut()
file_list = checkout.get_new_file()
if file_list is None:
logging.error("checkout files fail")
sys.exit(1)
# use cppcheck
cpp_check = CPPCheck(file_list)
cpp_check_result = cpp_check.check()
if not cpp_check_result:
logging.error("static code analysis(cppcheck) fail.")
sys.exit(1)
logging.info("check success.")
sys.exit(0)
if __name__ == '__main__':
cli()

292
rt-thread/tools/ci/file_check.py Normal file
View File

@@ -0,0 +1,292 @@
#
# Copyright (c) 2006-2022, RT-Thread Development Team
#
# SPDX-License-Identifier: Apache-2.0
#
# Change Logs:
# Date Author Notes
# 2021-04-01 LiuKang the first version
#
import os
import re
import sys
import click
import yaml
import chardet
import logging
import datetime
def init_logger():
log_format = "[%(filename)s %(lineno)d %(levelname)s] %(message)s "
date_format = '%Y-%m-%d %H:%M:%S %a '
logging.basicConfig(level=logging.INFO,
format=log_format,
datefmt=date_format,
)
class CheckOut:
def __init__(self, rtt_repo, rtt_branch):
self.root = os.getcwd()
self.rtt_repo = rtt_repo
self.rtt_branch = rtt_branch
def __exclude_file(self, file_path):
dir_number = file_path.split('/')
ignore_path = file_path
# gets the file path depth.
for i in dir_number:
# current directory.
dir_name = os.path.dirname(ignore_path)
ignore_path = dir_name
# judge the ignore file exists in the current directory.
ignore_file_path = os.path.join(dir_name, ".ignore_format.yml")
if not os.path.exists(ignore_file_path):
continue
try:
with open(ignore_file_path) as f:
ignore_config = yaml.safe_load(f.read())
file_ignore = ignore_config.get("file_path", [])
dir_ignore = ignore_config.get("dir_path", [])
except Exception as e:
logging.error(e)
continue
logging.debug("ignore file path: {}".format(ignore_file_path))
logging.debug("file_ignore: {}".format(file_ignore))
logging.debug("dir_ignore: {}".format(dir_ignore))
try:
# judge file_path in the ignore file.
for file in file_ignore:
if file is not None:
file_real_path = os.path.join(dir_name, file)
if file_real_path == file_path:
logging.info("ignore file path: {}".format(file_real_path))
return 0
file_dir_path = os.path.dirname(file_path)
for _dir in dir_ignore:
if _dir is not None:
dir_real_path = os.path.join(dir_name, _dir)
if file_dir_path.startswith(dir_real_path):
logging.info("ignore dir path: {}".format(dir_real_path))
return 0
except Exception as e:
logging.error(e)
continue
return 1
def get_new_file(self):
file_list = list()
try:
os.system('git remote add rtt_repo {}'.format(self.rtt_repo))
os.system('git fetch rtt_repo')
os.system('git merge rtt_repo/{}'.format(self.rtt_branch))
os.system('git reset rtt_repo/{} --soft'.format(self.rtt_branch))
os.system('git status > git.txt')
except Exception as e:
logging.error(e)
return None
try:
with open('git.txt', 'r') as f:
file_lines = f.readlines()
except Exception as e:
logging.error(e)
return None
file_path = ''
for line in file_lines:
if 'new file' in line:
file_path = line.split('new file:')[1].strip()
logging.info('new file -> {}'.format(file_path))
elif 'deleted' in line:
logging.info('deleted file -> {}'.format(line.split('deleted:')[1].strip()))
elif 'modified' in line:
file_path = line.split('modified:')[1].strip()
logging.info('modified file -> {}'.format(file_path))
else:
continue
result = self.__exclude_file(file_path)
if result != 0:
file_list.append(file_path)
return file_list
class FormatCheck:
def __init__(self, file_list):
self.file_list = file_list
def __check_rt_errorcode(self, line):
pattern = re.compile(r'return\s+(RT_ERROR|RT_ETIMEOUT|RT_EFULL|RT_EEMPTY|RT_ENOMEM|RT_ENOSYS|RT_EBUSY|RT_EIO|RT_EINTR|RT_EINVAL|RT_ENOENT|RT_ENOSPC|RT_EPERM|RT_ETRAP|RT_EFAULT)')
match = pattern.search(line)
if match:
return False
else:
return True
def __check_file(self, file_lines, file_path):
line_num = 0
check_result = True
for line in file_lines:
line_num += 1
# check line start
line_start = line.replace(' ', '')
# find tab
if line_start.startswith('\t'):
logging.error("{} line[{}]: please use space replace tab at the start of this line.".format(file_path, line_num))
check_result = False
# check line end
line_end = line.split('\n')[0]
if line_end.endswith(' ') or line_end.endswith('\t'):
logging.error("{} line[{}]: please delete extra space at the end of this line.".format(file_path, line_num))
check_result = False
if self.__check_rt_errorcode(line) == False:
logging.error("{} line[{}]: the RT-Thread error code should return negative value. e.g. return -RT_ERROR".format(file_path, line_num))
check_result = False
return check_result
def check(self):
logging.info("Start to check files format.")
if len(self.file_list) == 0:
logging.warning("There are no files to check format.")
return True
encoding_check_result = True
format_check_fail_files = 0
for file_path in self.file_list:
code = ''
if file_path.endswith(".c") or file_path.endswith(".h"):
try:
with open(file_path, 'rb') as f:
file = f.read()
# get file encoding
chardet_report = chardet.detect(file)
code = chardet_report['encoding']
confidence = chardet_report['confidence']
except Exception as e:
logging.error(e)
else:
continue
if code != 'utf-8' and code != 'ascii' and confidence > 0.8:
logging.error("[{0}]: encoding {1} not utf-8, please format it.".format(file_path, code))
encoding_check_result = False
else:
logging.info('[{0}]: encoding check success.'.format(file_path))
with open(file_path, 'r', encoding = "utf-8") as f:
file_lines = f.readlines()
if not self.__check_file(file_lines, file_path):
format_check_fail_files += 1
if (not encoding_check_result) or (format_check_fail_files != 0):
logging.error("files format check fail.")
return False
logging.info("files format check success.")
return True
class LicenseCheck:
def __init__(self, file_list):
self.file_list = file_list
def check(self):
current_year = datetime.date.today().year
logging.info("current year: {}".format(current_year))
if len(self.file_list) == 0:
logging.warning("There are no files to check license.")
return 0
logging.info("Start to check files license.")
check_result = True
for file_path in self.file_list:
if file_path.endswith(".c") or file_path.endswith(".h"):
try:
with open(file_path, 'r') as f:
file = f.readlines()
except Exception as e:
logging.error(e)
else:
continue
if 'Copyright' in file[1] and 'SPDX-License-Identifier: Apache-2.0' in file[3]:
try:
license_year = re.search(r'2006-\d{4}', file[1]).group()
true_year = '2006-{}'.format(current_year)
if license_year != true_year:
logging.warning("[{0}]: license year: {} is not true: {}, please update.".format(file_path,
license_year,
true_year))
else:
logging.info("[{0}]: license check success.".format(file_path))
except Exception as e:
logging.error(e)
else:
logging.error("[{0}]: license check fail.".format(file_path))
check_result = False
return check_result
@click.group()
@click.pass_context
def cli(ctx):
pass
@cli.command()
@click.option(
'--license',
"check_license",
required=False,
type=click.BOOL,
flag_value=True,
help="Enable File license check.",
)
@click.argument(
'repo',
nargs=1,
type=click.STRING,
default='https://github.com/RT-Thread/rt-thread',
)
@click.argument(
'branch',
nargs=1,
type=click.STRING,
default='master',
)
def check(check_license, repo, branch):
"""
check files license and format.
"""
init_logger()
# get modified files list
checkout = CheckOut(repo, branch)
file_list = checkout.get_new_file()
if file_list is None:
logging.error("checkout files fail")
sys.exit(1)
# check modified files format
format_check = FormatCheck(file_list)
format_check_result = format_check.check()
license_check_result = True
if check_license:
license_check = LicenseCheck(file_list)
license_check_result = license_check.check()
if not format_check_result or not license_check_result:
logging.error("file format check or license check fail.")
sys.exit(1)
logging.info("check success.")
sys.exit(0)
if __name__ == '__main__':
cli()

84
rt-thread/tools/ci/format_ignore.py Normal file
View File

@@ -0,0 +1,84 @@
#
# Copyright (c) 2006-2023, RT-Thread Development Team
#
# SPDX-License-Identifier: Apache-2.0
#
# Change Logs:
# Date Author Notes
# 2023-05-16 dejavudwh the first version
#
import yaml
import logging
import os
import subprocess
def init_logger():
log_format = "[%(filename)s %(lineno)d %(levelname)s] %(message)s "
date_format = '%Y-%m-%d %H:%M:%S %a '
logging.basicConfig(level=logging.INFO,
format=log_format,
datefmt=date_format,
)
class CheckOut:
def __init__(self):
pass
def __exclude_file(self, file_path):
dir_number = file_path.split('/')
ignore_path = file_path
# gets the file path depth.
for i in dir_number:
# current directory.
dir_name = os.path.dirname(ignore_path)
ignore_path = dir_name
# judge the ignore file exists in the current directory.
ignore_file_path = os.path.join(dir_name, ".ignore_format.yml")
if not os.path.exists(ignore_file_path):
continue
try:
with open(ignore_file_path) as f:
ignore_config = yaml.safe_load(f.read())
file_ignore = ignore_config.get("file_path", [])
dir_ignore = ignore_config.get("dir_path", [])
except Exception as e:
logging.error(e)
continue
logging.debug("ignore file path: {}".format(ignore_file_path))
logging.debug("file_ignore: {}".format(file_ignore))
logging.debug("dir_ignore: {}".format(dir_ignore))
try:
# judge file_path in the ignore file.
for file in file_ignore:
if file is not None:
file_real_path = os.path.join(dir_name, file)
if file_real_path == file_path:
logging.info("ignore file path: {}".format(file_real_path))
return 0
file_dir_path = os.path.dirname(file_path)
for _dir in dir_ignore:
if _dir is not None:
dir_real_path = os.path.join(dir_name, _dir)
if file_dir_path.startswith(dir_real_path):
logging.info("ignore dir path: {}".format(dir_real_path))
return 0
except Exception as e:
logging.error(e)
continue
return 1
def get_new_file(self):
result = subprocess.run(['git', 'diff', '--name-only', 'HEAD', 'origin/master', '--diff-filter=ACMR', '--no-renames', '--full-index'], stdout = subprocess.PIPE)
file_list = result.stdout.decode().strip().split('\n')
new_files = []
for line in file_list:
logging.info("modified file -> {}".format(line))
result = self.__exclude_file(line)
if result != 0:
new_files.append(line)
return new_files