From 49773c662af38fb32ebf1ad9d48151b9fef75f0b Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Mon, 18 Dec 2023 20:01:27 +0700 Subject: [PATCH 1/3] fix(idf_tools): Fix platform detection for arm64 machine but arm32 environment --- tools/idf_tools.py | 51 +++++++++++++----- .../platform_detection/arm32_header.elf | Bin 0 -> 52 bytes .../platform_detection/arm64_header.elf | Bin 0 -> 64 bytes .../platform_detection/armhf_header.elf | Bin 0 -> 52 bytes tools/test_idf_tools/test_idf_tools.py | 27 +++++++++- 5 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 tools/test_idf_tools/platform_detection/arm32_header.elf create mode 100644 tools/test_idf_tools/platform_detection/arm64_header.elf create mode 100644 tools/test_idf_tools/platform_detection/armhf_header.elf diff --git a/tools/idf_tools.py b/tools/idf_tools.py index d4212cb3f6..4f67ac9df3 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -164,6 +164,42 @@ class Platforms: 'Linux-arm': PLATFORM_LINUX_ARM32, } + @staticmethod + def detect_linux_arm_platform(supposed_platform): # type: (Optional[str]) -> Optional[str] + """ + We probe the python binary to check exactly what environment the script is running in. + + ARM platform may run on armhf hardware but having armel installed packages. + To avoid possible armel/armhf libraries mixing need to define user's + packages architecture to use the same + See note section in https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html#index-mfloat-abi + + ARM platform may run on aarch64 hardware but having armhf installed packages + (it happens if a docker container is running on arm64 hardware, but using an armhf image). + + """ + if supposed_platform not in (PLATFORM_LINUX_ARM32, PLATFORM_LINUX_ARMHF, PLATFORM_LINUX_ARM64): + return supposed_platform + + # suppose that installed python was built with the right ABI + with open(sys.executable, 'rb') as f: + # see ELF header description in https://man7.org/linux/man-pages/man5/elf.5.html, offsets depend on ElfN size + if int.from_bytes(f.read(4), sys.byteorder) != int.from_bytes(b'\x7fELF', sys.byteorder): + return supposed_platform # ELF magic not found. Use the default platform name from PLATFORM_FROM_NAME + f.seek(18) # seek to e_machine + e_machine = int.from_bytes(f.read(2), sys.byteorder) + if e_machine == 183: # EM_AARCH64, https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst + supposed_platform = PLATFORM_LINUX_ARM64 + elif e_machine == 40: # EM_ARM, https://github.com/ARM-software/abi-aa/blob/main/aaelf32/aaelf32.rst + f.seek(36) # seek to e_flags + e_flags = int.from_bytes(f.read(4), sys.byteorder) + if e_flags & 0x400: + supposed_platform = PLATFORM_LINUX_ARMHF + else: + supposed_platform = PLATFORM_LINUX_ARM32 + + return supposed_platform + @staticmethod def get(platform_alias): # type: (Optional[str]) -> Optional[str] if platform_alias is None: @@ -171,21 +207,8 @@ class Platforms: if platform_alias == 'any' and CURRENT_PLATFORM: platform_alias = CURRENT_PLATFORM - platform_name = Platforms.PLATFORM_FROM_NAME.get(platform_alias, None) - - # ARM platform may run on armhf hardware but having armel installed packages. - # To avoid possible armel/armhf libraries mixing need to define user's - # packages architecture to use the same - # See note section in https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html#index-mfloat-abi - if platform_name in (PLATFORM_LINUX_ARM32, PLATFORM_LINUX_ARMHF) and 'arm' in platform.machine(): - # suppose that installed python was built with a right ABI - with open(sys.executable, 'rb') as f: - if int.from_bytes(f.read(4), sys.byteorder) != int.from_bytes(b'\x7fELF', sys.byteorder): - return platform_name # ELF magic not found. Use default platform name from PLATFORM_FROM_NAME - f.seek(36) # seek to e_flags (https://man7.org/linux/man-pages/man5/elf.5.html) - e_flags = int.from_bytes(f.read(4), sys.byteorder) - platform_name = PLATFORM_LINUX_ARMHF if e_flags & 0x400 else PLATFORM_LINUX_ARM32 + platform_name = Platforms.detect_linux_arm_platform(platform_name) return platform_name @staticmethod diff --git a/tools/test_idf_tools/platform_detection/arm32_header.elf b/tools/test_idf_tools/platform_detection/arm32_header.elf new file mode 100644 index 0000000000000000000000000000000000000000..d1ac728eac59a262414d8ddc09fc5238dc7a66a3 GIT binary patch literal 52 ycmb<-^>JflWMpQ50%isc21X$JMkgDC36SQvdWwO8iGkIGL4kn_C@0S##{dAECIyuM literal 0 HcmV?d00001 diff --git a/tools/test_idf_tools/platform_detection/arm64_header.elf b/tools/test_idf_tools/platform_detection/arm64_header.elf new file mode 100644 index 0000000000000000000000000000000000000000..44e5808390e17a1dc782b60b6c9b182a46f064b4 GIT binary patch literal 64 ycmb<-^>JfjWMpQ50%nHo42(ebmF2u(5eKOFfql23ECvP#1`7r*ASurv#{dBNVh1?@ literal 0 HcmV?d00001 diff --git a/tools/test_idf_tools/platform_detection/armhf_header.elf b/tools/test_idf_tools/platform_detection/armhf_header.elf new file mode 100644 index 0000000000000000000000000000000000000000..7531b0538db4c0f2ce0ad04a265ba700fe3935ad GIT binary patch literal 52 ycmb<-^>JflWMpQ50%isc21X$JX9*jF36TEsrJI3)g@M(CL4kn_C@0S##{dAV_y%tP literal 0 HcmV?d00001 diff --git a/tools/test_idf_tools/test_idf_tools.py b/tools/test_idf_tools/test_idf_tools.py index 988efdbbbf..c90719fa29 100755 --- a/tools/test_idf_tools/test_idf_tools.py +++ b/tools/test_idf_tools/test_idf_tools.py @@ -2,7 +2,6 @@ # # SPDX-FileCopyrightText: 2019-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - import json import os import re @@ -10,6 +9,7 @@ import shutil import sys import tempfile import unittest +from unittest.mock import patch try: from contextlib import redirect_stdout @@ -650,5 +650,30 @@ class TestMaintainer(unittest.TestCase): self.assertEqual(json.load(f1), expected_json, "Please check 'tools/tools.new.json' to find a cause!") +class TestArmDetection(unittest.TestCase): + + ELF_HEADERS = { + idf_tools.PLATFORM_LINUX_ARM64: 'platform_detection/arm64_header.elf', + idf_tools.PLATFORM_LINUX_ARMHF: 'platform_detection/armhf_header.elf', + idf_tools.PLATFORM_LINUX_ARM32: 'platform_detection/arm32_header.elf', + } + + ARM_PLATFORMS = { + idf_tools.PLATFORM_LINUX_ARM64, + idf_tools.PLATFORM_LINUX_ARMHF, + idf_tools.PLATFORM_LINUX_ARM32, + } + + def test_arm_detection(self): + for platform in idf_tools.Platforms.PLATFORM_FROM_NAME.values(): + with patch('sys.executable', __file__): # use invalid ELF as executable. In this case passed parameter must return + self.assertEqual(idf_tools.Platforms.detect_linux_arm_platform(platform), platform) + # detect_linux_arm_platform() intended to return arch that detected in sys.executable ELF + for exec_platform in (idf_tools.PLATFORM_LINUX_ARM64, idf_tools.PLATFORM_LINUX_ARMHF, idf_tools.PLATFORM_LINUX_ARM32): + with patch('sys.executable', TestArmDetection.ELF_HEADERS[exec_platform]): + for platform in TestArmDetection.ARM_PLATFORMS: + self.assertEqual(idf_tools.Platforms.detect_linux_arm_platform(platform), exec_platform) + + if __name__ == '__main__': unittest.main() From 2456dc6a8e6834e73ef0cdea38bbfa2cc110129f Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Wed, 10 Jan 2024 20:18:03 +0700 Subject: [PATCH 2/3] fix(idf_tools): fix delimiter when the updater parsing, ' *' or ' ' --- tools/idf_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 4f67ac9df3..2809fdc7aa 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -2372,7 +2372,7 @@ class ChecksumFileParser(): try: for bytes_str, hash_str in zip(self.checksum[0::2], self.checksum[1::2]): bytes_filename = self.parseLine(r'^# (\S*):', bytes_str) - hash_filename = self.parseLine(r'^\S* \*(\S*)', hash_str) + hash_filename = self.parseLine(r'^\S* [\* ](\S*)', hash_str) if hash_filename != bytes_filename: fatal('filename in hash-line and in bytes-line are not the same') raise SystemExit(1) From 61bc4b968493d22867167e8d1de95adad0a7f3ed Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Wed, 18 Sep 2024 11:11:26 +0700 Subject: [PATCH 3/3] ci: update mypy check for python 3.8 --- tools/ci/check_type_comments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci/check_type_comments.py b/tools/ci/check_type_comments.py index d9e6c6b9af..38865b0842 100755 --- a/tools/ci/check_type_comments.py +++ b/tools/ci/check_type_comments.py @@ -29,7 +29,7 @@ def types_valid_ignored_rules(file_name): # type: (str) -> bool """ Run Mypy check with rules for ignore list on the given file, return TRUE if Mypy check passes """ - mypy_exit_code = subprocess.call('mypy {} --python-version 3.7 --allow-untyped-defs'.format(file_name), shell=True) + mypy_exit_code = subprocess.call('mypy {} --python-version 3.8 --allow-untyped-defs'.format(file_name), shell=True) return not bool(mypy_exit_code)