Merge pull request #4567 from enkiller/0405

[tools] Python 3 compatibility support

tools/auto-ci.py

@@ -52,7 +52,7 @@ fail = False
 
 BSP_ROOT = '../bsp'
 
-for bsp,cpu in bsp_to_cpu.iteritems():
+for bsp,cpu in bsp_to_cpu.items():
     project_dir = os.path.join(BSP_ROOT, bsp)
     if os.getenv('RTT_CPU') == cpu and os.path.isfile(os.path.join(project_dir, 'SConstruct')):
         if os.system('scons --directory=' + project_dir) != 0:
@@ -63,10 +63,10 @@ for bsp,cpu in bsp_to_cpu.iteritems():
     else:
         results['ignore'].append(bsp)
 
-for result,bsp_list in results.iteritems():
+for result,bsp_list in results.items():
-    print "## {0}: {1}\n".format(result, len(bsp_list))
+    print("## {0}: {1}\n".format(result, len(bsp_list)))
     for bsp in bsp_list:
-        print "* " + bsp
+        print("* " + bsp)
 
 if fail:
     sys.exit(1)

tools/buildbot.py

@@ -2,9 +2,9 @@ import os
 import sys
 
 def usage():
-    print '%s all     -- build all bsp' % os.path.basename(sys.argv[0])
+    print('%s all     -- build all bsp' % os.path.basename(sys.argv[0]))
-    print '%s clean   -- clean all bsp' % os.path.basename(sys.argv[0])
+    print('%s clean   -- clean all bsp' % os.path.basename(sys.argv[0]))
-    print '%s project -- update all prject files' % os.path.basename(sys.argv[0])
+    print('%s project -- update all prject files' % os.path.basename(sys.argv[0]))
 
 BSP_ROOT = os.path.join("..", "bsp")
 
@@ -25,25 +25,25 @@ elif sys.argv[1] == 'project':
         project_dir = os.path.join(BSP_ROOT, item)
 
         if os.path.isfile(os.path.join(project_dir, 'template.Uv2')):
-            print ('prepare MDK3 project file on ' + project_dir)
+            print('prepare MDK3 project file on ' + project_dir)
             command = ' --target=mdk -s'
 
             os.system('scons --directory=' + project_dir + command)
 
         if os.path.isfile(os.path.join(project_dir, 'template.uvproj')):
-            print ('prepare MDK4 project file on ' + project_dir)
+            print('prepare MDK4 project file on ' + project_dir)
             command = ' --target=mdk4 -s'
 
             os.system('scons --directory=' + project_dir + command)
 
         if os.path.isfile(os.path.join(project_dir, 'template.uvprojx')):
-            print ('prepare MDK5 project file on ' + project_dir)
+            print('prepare MDK5 project file on ' + project_dir)
             command = ' --target=mdk5 -s'
 
             os.system('scons --directory=' + project_dir + command)
 
         if os.path.isfile(os.path.join(project_dir, 'template.ewp')):
-            print ('prepare IAR project file on ' + project_dir)
+            print('prepare IAR project file on ' + project_dir)
             command = ' --target=iar -s'
 
             os.system('scons --directory=' + project_dir + command)
@@ -58,5 +58,5 @@ for item in projects:
     project_dir = os.path.join(BSP_ROOT, item)
     if os.path.isfile(os.path.join(project_dir, 'SConstruct')):
         if os.system('scons --directory=' + project_dir + command) != 0:
-            print 'build failed!!'
+            print('build failed!!')
             break

tools/building.py

@@ -85,7 +85,7 @@ class Win32Spawn:
                 try:
                     os.remove(f)
                 except Exception as e:
-                    print ('Error removing file: ' + e)
+                    print('Error removing file: ' + e)
                     return -1
             return 0
 
@@ -106,8 +106,8 @@ class Win32Spawn:
         try:
             proc = subprocess.Popen(cmdline, env=_e, shell=False)
         except Exception as e:
-            print ('Error in calling command:' + cmdline.split(' ')[0])
+            print('Error in calling command:' + cmdline.split(' ')[0])
-            print ('Exception: ' + os.strerror(e.errno))
+            print('Exception: ' + os.strerror(e.errno))
             if (os.strerror(e.errno) == "No such file or directory"):
                 print ("\nPlease check Toolchains PATH setting.\n")
 
@@ -274,7 +274,7 @@ def PrepareBuilding(env, root_directory, has_libcpu=False, remove_components = [
             os.environ['RTT_CC'] = rtconfig.CROSS_TOOL
             utils.ReloadModule(rtconfig)
         except KeyError:
-            print ('Unknow target: '+ tgt_name+'. Avaible targets: ' +', '.join(tgt_dict.keys()))
+            print('Unknow target: '+ tgt_name+'. Avaible targets: ' +', '.join(tgt_dict.keys()))
             sys.exit(1)
 
     # auto change the 'RTT_EXEC_PATH' when 'rtconfig.EXEC_PATH' get failed
@@ -662,7 +662,7 @@ def DefineGroup(name, src, depend, **parameters):
     # check whether to clean up library
     if GetOption('cleanlib') and os.path.exists(os.path.join(group['path'], GroupLibFullName(name, Env))):
         if group['src'] != []:
-            print ('Remove library:'+ GroupLibFullName(name, Env))
+            print('Remove library:'+ GroupLibFullName(name, Env))
             fn = os.path.join(group['path'], GroupLibFullName(name, Env))
             if os.path.exists(fn):
                 os.unlink(fn)
@@ -735,7 +735,7 @@ def BuildLibInstallAction(target, source, env):
         if Group['name'] == lib_name:
             lib_name = GroupLibFullName(Group['name'], env)
             dst_name = os.path.join(Group['path'], lib_name)
-            print ('Copy '+lib_name+' => ' +dst_name)
+            print('Copy '+lib_name+' => ' + dst_name)
             do_copy_file(lib_name, dst_name)
             break
 
@@ -996,11 +996,11 @@ def GetVersion():
     prepcessor.process_contents(contents)
     def_ns = prepcessor.cpp_namespace
 
-    version = int(filter(lambda ch: ch in '0123456789.', def_ns['RT_VERSION']))
+    version = int([ch for ch in def_ns['RT_VERSION'] if ch in '0123456789.'])
-    subversion = int(filter(lambda ch: ch in '0123456789.', def_ns['RT_SUBVERSION']))
+    subversion = int([ch for ch in def_ns['RT_SUBVERSION'] if ch in '0123456789.'])
 
     if 'RT_REVISION' in def_ns:
-        revision = int(filter(lambda ch: ch in '0123456789.', def_ns['RT_REVISION']))
+        revision = int([ch for ch in def_ns['RT_REVISION'] if ch in '0123456789.'])
         return '%d.%d.%d' % (version, subversion, revision)
 
     return '0.%d.%d' % (version, subversion)

tools/clang-analyze.py

@@ -53,9 +53,9 @@ def generate(env):
 
     if rtconfig.EXEC_PATH:
         if not os.path.exists(rtconfig.EXEC_PATH):
-            print
+            print()
-            print 'warning: rtconfig.EXEC_PATH(%s) does not exists.' % rtconfig.EXEC_PATH
+            print('warning: rtconfig.EXEC_PATH(%s) does not exists.' % rtconfig.EXEC_PATH)
-            print
+            print()
             return
         env.AppendENVPath('PATH', rtconfig.EXEC_PATH)
 

tools/eclipse.py

@@ -212,7 +212,7 @@ def HandleToolOption(tools, env, project, reset):
                     linker_nostart_option = option
                 elif option.get('id').find('linker.libs') != -1:
                     linker_libs_option = option
-                elif option.get('id').find('linker.paths') != -1 and env.has_key('LIBPATH'):
+                elif option.get('id').find('linker.paths') != -1 and 'LIBPATH' in env:
                     linker_paths_option = option
                 elif option.get('id').find('linker.usenewlibnano') != -1:
                     linker_newlib_nano_option = option
@@ -317,7 +317,7 @@ def HandleToolOption(tools, env, project, reset):
                 option.remove(item)
 
         # add new libs
-        if env.has_key('LIBS'):
+        if 'LIBS' in env:
             for lib in env['LIBS']:
                 formatedLib = ConverToRttEclipseLibFormat(lib)
                 SubElement(option, 'listOptionValue', {

tools/keil.py

@@ -122,12 +122,18 @@ def MDK4AddLibToGroup(ProjectFiles, group, name, filename, project_path):
     if ProjectFiles.count(obj_name):
         name = basename + '_' + name
     ProjectFiles.append(obj_name)
-    file_name.text = name.decode(fs_encoding)
+    try:
+        file_name.text = name.decode(fs_encoding)
+    except:
+        file_name.text = name
     file_type = SubElement(file, 'FileType')
     file_type.text = '%d' % _get_filetype(name)
     file_path = SubElement(file, 'FilePath')
 
-    file_path.text = path.decode(fs_encoding)
+    try:
+        file_path.text = path.decode(fs_encoding)
+    except:
+        file_path.text = path
 
     return group
 

tools/mkromfs.py

@@ -5,7 +5,7 @@ import os
 
 import struct
 from collections import namedtuple
-import StringIO
+import io
 
 import argparse
 parser = argparse.ArgumentParser()
@@ -43,8 +43,10 @@ class File(object):
 
         if self.entry_size == 0:
             return ''
-
+        if len(self._data) > 0 and type(self._data[0]) == int:
-        return head + ','.join(('0x%02x' % ord(i) for i in self._data)) + tail
+            return head + ','.join(('0x%02x' % i for i in self._data)) + tail
+        else:
+            return head + ','.join(('0x%02x' % ord(i) for i in self._data)) + tail
 
     @property
     def entry_size(self):
@@ -104,7 +106,8 @@ class Folder(object):
                 return 1
             else:
                 return -1
-        self._children.sort(cmp=_sort)
+        from functools import cmp_to_key
+        self._children.sort(key=cmp_to_key(_sort))
 
         # sort recursively
         for c in self._children:
@@ -255,7 +258,7 @@ if __name__ == '__main__':
     if args.binary:
         data = get_bin_data(tree, int(args.addr, 16))
     else:
-        data = get_c_data(tree)
+        data = get_c_data(tree).encode()
 
     output = args.output
     if not output:

tools/ses.py

@@ -59,14 +59,14 @@ def SESProject(env) :
         group_tree = SDKAddGroup(project_node, group['name'], group['src'], project_path)
 
         # get each group's cc flags
-        if group.has_key('CCFLAGS') and group['CCFLAGS']:
+        if 'CCFLAGS' in group and group['CCFLAGS']:
             if CCFLAGS:
                 CCFLAGS += ' ' + group['CCFLAGS']
             else:
                 CCFLAGS += group['CCFLAGS']   
                 
         # get each group's link flags
-        if group.has_key('LINKFLAGS') and group['LINKFLAGS']:
+        if 'LINKFLAGS' in group and group['LINKFLAGS']:
             if LINKFLAGS:
                 LINKFLAGS += ' ' + group['LINKFLAGS']
             else:

tools/ua.py

@@ -45,7 +45,7 @@ def PrepareUA(project, RTT_ROOT, BSP_ROOT):
         # ua.write('import sys\n')
         ua.write('\n')
         
-        print RTT_ROOT
+        print(RTT_ROOT)
         
         CPPPATH = []
         CPPDEFINES = []

tools/vs.py

@@ -45,9 +45,12 @@ def VS_AddGroup(ProjectFiles, parent, name, files, libs, project_path):
 
         path = _make_path_relative(project_path, path)
         path = os.path.join(path, name)
-
+        try:
+            path = path.decode(fs_encoding)
+        except:
+            path = path
         File = SubElement(Filter, 'File')
-        File.set('RelativePath', path.decode(fs_encoding))
+        File.set('RelativePath', path)
 
     for lib in libs:
         name = os.path.basename(lib)
@@ -57,7 +60,11 @@ def VS_AddGroup(ProjectFiles, parent, name, files, libs, project_path):
         path = os.path.join(path, name)
 
         File = SubElement(Filter, 'File')
-        File.set('RelativePath', path.decode(fs_encoding))
+        try:
+            path = path.decode(fs_encoding)
+        except:
+            path = path
+        File.set('RelativePath', path)
 
 def VS_AddHeadFilesGroup(program, elem, project_path):
     utils.source_ext = []
@@ -70,7 +77,11 @@ def VS_AddHeadFilesGroup(program, elem, project_path):
     for f in utils.source_list:
         path = _make_path_relative(project_path, f)
         File = SubElement(elem, 'File')
-        File.set('RelativePath', path.decode(fs_encoding))
+        try:
+            path = path.decode(fs_encoding)
+        except:
+            path = path
+        File.set('RelativePath', path)
 
 def VSProject(target, script, program):
     project_path = os.path.dirname(os.path.abspath(target))
@@ -158,12 +169,17 @@ def VSProject(target, script, program):
         for path in lib_path:
             inc = _make_path_relative(project_path, os.path.normpath(path))
             paths.add(inc) #.replace('\\', '/')
-    
+
         paths = [i for i in paths]
         paths.sort()
         lib_paths = ';'.join(paths)
         elem.set('AdditionalLibraryDirectories', lib_paths)
 
     xml_indent(root)
-    out.write(etree.tostring(root, encoding='utf-8'))
+    text = etree.tostring(root, encoding='utf-8')
+    try:
+        text = text.decode(encoding="utf-8")
+    except:
+        text = text
+    out.write(text)
     out.close()

tools/vs2012.py

@@ -44,7 +44,12 @@ fs_encoding = sys.getfilesystemencoding()
 filter_project = etree.Element('Project', attrib={'ToolsVersion':'4.0'})
 def get_uuid():
     id = uuid.uuid1()  # UUID('3e5526c0-2841-11e3-a376-20cf3048bcb3')
-    idstr = id.get_urn()[9:] #'urn:uuid:3e5526c0-2841-11e3-a376-20cf3048bcb3'[9:]
+    if sys.version > '3':
+        idstr = id.urn[9:] #'urn:uuid:3e5526c0-2841-11e3-a376-20cf3048bcb3'[9:]
+    else:
+        # python3 is no decode function
+        idstr = id.get_urn()[9:] #'urn:uuid:3e5526c0-2841-11e3-a376-20cf3048bcb3'[9:]
+    
     return '{'+idstr+'}'
 
 def VS2012_AddGroup(parent, group_name, files, project_path):
@@ -57,7 +62,12 @@ def VS2012_AddGroup(parent, group_name, files, project_path):
         path = os.path.join(path, name)
 
         ClCompile = SubElement(parent, 'ClCompile')
-        ClCompile.set('Include', path.decode(fs_encoding))
+
+        if sys.version > '3':
+            ClCompile.set('Include', path)
+        else:
+            # python3 is no decode function
+            ClCompile.set('Include', path.decode(fs_encoding))
 
         Filter = SubElement(ClCompile, 'Filter')
         Filter.text='Source Files\\'+group_name
@@ -119,7 +129,13 @@ def VS_add_ItemGroup(parent, file_type, files, project_path):
         path = os.path.join(path, name)
 
         File = SubElement(ItemGroup, item_tag)
-        File.set('Include', path.decode(fs_encoding))
+
+        if sys.version > '3':
+            File.set('Include', path)
+        else:
+            # python3 is no decode function
+            File.set('Include', path.decode(fs_encoding))
+
         if file_type == 'C' :
             ObjName = SubElement(File, 'ObjectFileName')
             ObjName.text = ''.join('$(IntDir)'+objpath+'\\')
@@ -137,11 +153,22 @@ def VS_add_HeadFiles(program, elem, project_path):
     for f in utils.source_list:
         path = _make_path_relative(project_path, f)
         File = SubElement(ItemGroup, 'ClInclude')
-        File.set('Include', path.decode(fs_encoding))
+
+        if sys.version > '3':
+            File.set('Include', path)
+        else:
+            # python3 is no decode function
+            File.set('Include', path.decode(fs_encoding))
 
         # add project.vcxproj.filter
         ClInclude = SubElement(filter_h_ItemGroup, 'ClInclude')
-        ClInclude.set('Include', path.decode(fs_encoding))
+
+        if sys.version > '3':
+            ClInclude.set('Include', path)
+        else:
+            # python3 is no decode function
+            ClInclude.set('Include', path.decode(fs_encoding))
+
         Filter = SubElement(ClInclude, 'Filter')
         Filter.text='Header Files'
 
@@ -151,12 +178,12 @@ def VS2012Project(target, script, program):
     tree = etree.parse('template_vs2012.vcxproj')
     root = tree.getroot()
     elem = root
-    
+
-    out = file(target, 'wb')
+    out = open(target, 'w')
     out.write('<?xml version="1.0" encoding="UTF-8"?>\r\n')
-    
+
     ProjectFiles = []
-    
+
     # add "*.c or *.h" files
 
     VS2012_CreateFilter(script, project_path)
@@ -187,7 +214,16 @@ def VS2012Project(target, script, program):
     # write cppdefinitons flags
     if 'CPPDEFINES' in building.Env:
         for elem in tree.iter(tag='PreprocessorDefinitions'):
-            definitions = ';'.join(building.Env['CPPDEFINES']) + ';%(PreprocessorDefinitions)'
+            CPPDEFINES = building.Env['CPPDEFINES']
+            definitions = []
+            if type(CPPDEFINES[0]) == type(()):
+                for item in CPPDEFINES:
+                    definitions += [i for i in item]
+                definitions = ';'.join(definitions)
+            else:
+                definitions = ';'.join(building.Env['CPPDEFINES'])
+
+            definitions = definitions + ';%(PreprocessorDefinitions)'
             elem.text = definitions
             break
     # write link flags
@@ -216,15 +252,27 @@ def VS2012Project(target, script, program):
             break
 
     xml_indent(root)
-    vcxproj_string = etree.tostring(root, encoding='utf-8')
+
+    if sys.version > '3':
+        vcxproj_string = etree.tostring(root, encoding='unicode')
+    else:
+        # python3 is no decode function
+        vcxproj_string = etree.tostring(root, encoding='utf-8')
+
     root_node=r'<Project DefaultTargets="Build" ToolsVersion="4.0">'
     out.write(r'<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">')
     out.write(vcxproj_string[len(root_node):])
     out.close()
 
     xml_indent(filter_project)
-    filter_string = etree.tostring(filter_project, encoding='utf-8')
+
-    out = file('project.vcxproj.filters', 'wb')
+    if sys.version > '3':
+        filter_string = etree.tostring(filter_project, encoding='unicode')
+    else:
+        # python3 is no decode function
+        filter_string = etree.tostring(filter_project, encoding='utf-8')
+
+    out = open('project.vcxproj.filters', 'w')
     out.write('<?xml version="1.0" encoding="UTF-8"?>\r\n')
     root_node=r'<Project ToolsVersion="4.0">'
     out.write(r'<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">')

tools/win32spawn.py

@@ -24,7 +24,13 @@
 
 import os
 import threading
-import Queue
+import sys
+
+_PY2 = sys.version_info[0] < 3
+if _PY2:
+    import Queue
+else:
+    import queue as Queue
 
 # Windows import 
 import win32file