espressif/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-04-27 19:13:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ldgen): place input sections from mutable libraries into mutable markers

Browse Source 
Introduce a new `--mutable-libraries-file` option that accepts a file
containing the filenames of mutable libraries, each listed on a separate
line. Mutable libraries are component libraries expected to change
during development. In contrast, immutable component libraries are not
expected to change. In the generated linker script, the input sections
of mutable libraries are grouped together rather than being mixed with
those of immutable libraries. The goal is to create large continuous
areas in the ELF file's output sections that remain unchanged for
immutable libraries during application recompilation, allowing these
areas to be skipped during flashing.

The build system identifies the mutable libraries and passes them to
ldgen using the `--mutable-libraries-file` option. It maintains
information about component sources, one of which is
`project_components`. This source type identifies components that are
directly related to the project being developed and are very likely to
change.

Mappings for mutable libraries are explicitly created for all sections
in the default scheme. This happens before the entity
`(archive:object_file:symbol/input_section)` node tree with placements is
generated and is equivalent to having these mappings in the mapping
linker fragment. All placements for mappings, whether newly added or
already existing as defined in linker fragments, associated with mutable
libraries are flagged as `mutable` in the entity node tree. This flag
ensures that these placements are included in the final linker script.
Currently, ldgen only emits placements that are either significant or
forced. A placement is considered significant if, for example, it is not
already covered by a placement in parent node. For instance, `*(.iram1
.iram1.*)` placement already includes `*libapp_trace.a:(.iram1
.iram1.*)`, so the latter is not emitted by default. The `mutable` flag
ensures that placements for mutable libraries are emitted in the linker
script and placed at dedicated location.

The locations where placements for mutable libraries are specified in
the linker script are identified by a new `mutable` marker, for example,
`mutable[flash_text]`. The placements for immutable libraries remain in
the existing `mapping` marker, for example, `mapping[flash_text]`. The
`mutable` marker for each target is placed after the `mapping` marker.

Signed-off-by: Frantisek Hrbata <frantisek.hrbata@espressif.com>
This commit is contained in:
Frantisek Hrbata
2025-05-22 14:17:32 +02:00
parent 2ade22ff85
commit 299e172fca
4 changed files with 115 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/ldgen/ldgen.py
+8 -1
View File
@@ -62,6 +62,11 @@ def main():
type=argparse.FileType('r'),
help='File that contains the list of libraries in the build')
argparser.add_argument(
'--mutable-libraries-file',
type=argparse.FileType('r'),
help='File that contains the list of mutable libraries in the build')
argparser.add_argument(
'--output', '-o',
help='Output linker script',
@@ -104,6 +109,7 @@ def main():
input_file = args.input
libraries_file = args.libraries_file
mutable_libraries_file = args.mutable_libraries_file or []
config_file = args.config
output_path = args.output
kconfig_file = args.kconfig
@@ -132,7 +138,8 @@ def main():
dump.name = library
sections_infos.add_sections_info(dump)
generation_model = Generation(check_mapping, check_mapping_exceptions)
mutable_libs = [lib.strip() for lib in mutable_libraries_file]
generation_model = Generation(check_mapping, check_mapping_exceptions, mutable_libs, args.debug)
_update_environment(args) # assign args.env and args.env_file to os.environ
tools/ldgen/ldgen/generation.py
+93 -29
View File
@@ -41,7 +41,7 @@ class Placement:
for details).
"""
def __init__(self, node, sections, target, flags, explicit, force=False, dryrun=False):
def __init__(self, node, sections, target, flags, explicit, force=False, dryrun=False, mutable=False):
self.node = node
self.sections = sections
self.target = target
@@ -57,6 +57,10 @@ class Placement:
# fragment entry.
self.explicit = explicit
# This placement is designated as mutable and should be assigned to a
# mutable target.
self.mutable = mutable
# Find basis placement.
parent = node.parent
candidate = None
@@ -83,10 +87,16 @@ class Placement:
#
# Placement can also be a basis if it has flags
# (self.flags) or its basis has flags (self.basis.flags)
#
# The mutable placement mark is used to place the placement within a
# designated area (MutableMarker) in the linker script. If the basis is
# also mutable, the placement is deemed insignificant to prevent adding
# unnecessary EXCLUDE_FILE and placement rules to the linker script.
significant = (not self.basis or
self.target != self.basis.target or
(self.flags and not self.basis.flags) or
(not self.flags and self.basis.flags) or
(self.mutable and not self.basis.mutable) or
self.force)
if significant and not self.explicit and not self.sections:
@@ -112,7 +122,6 @@ class Placement:
def add_subplacement(self, subplacement):
self.subplacements.add(subplacement)
class EntityNode:
"""
Node in entity tree. An EntityNode
@@ -196,6 +205,7 @@ class EntityNode:
keep = False
sort = None
surround_type = []
mutable = placement.mutable
placement_flags = placement.flags if placement.flags is not None else []
@@ -212,9 +222,9 @@ class EntityNode:
for flag in surround_type:
if flag.pre:
if isinstance(flag, Surround):
commands[placement.target].append(SymbolAtAddress(f'_{flag.symbol}_start', tied))
commands[placement.target].append(SymbolAtAddress(f'_{flag.symbol}_start', tied, mutable=mutable))
else: # ALIGN
commands[placement.target].append(AlignAtAddress(flag.alignment, tied))
commands[placement.target].append(AlignAtAddress(flag.alignment, tied, mutable=mutable))
# This is for expanded object node and symbol node placements without checking for
# the type.
@@ -222,7 +232,7 @@ class EntityNode:
command_sections = sections if sections == placement_sections else placement_sections
command = InputSectionDesc(placement.node.entity, command_sections,
[e.node.entity for e in placement.exclusions], keep, sort, tied)
[e.node.entity for e in placement.exclusions], keep, sort, tied, mutable=mutable)
commands[placement.target].append(command)
# Generate commands for intermediate, non-explicit exclusion placements here,
@@ -230,34 +240,34 @@ class EntityNode:
for subplacement in placement.subplacements:
if not subplacement.flags and not subplacement.explicit:
command = InputSectionDesc(subplacement.node.entity, subplacement.sections,
[e.node.entity for e in subplacement.exclusions], keep, sort, tied)
[e.node.entity for e in subplacement.exclusions], keep, sort, tied, mutable=mutable)
commands[placement.target].append(command)
for flag in surround_type:
if flag.post:
if isinstance(flag, Surround):
commands[placement.target].append(SymbolAtAddress(f'_{flag.symbol}_end', tied))
commands[placement.target].append(SymbolAtAddress(f'_{flag.symbol}_end', tied, mutable=mutable))
else: # ALIGN
commands[placement.target].append(AlignAtAddress(flag.alignment, tied))
commands[placement.target].append(AlignAtAddress(flag.alignment, tied, mutable=mutable))
return commands
def self_placement(self, sections, target, flags, explicit=True, force=False):
placement = Placement(self, sections, target, flags, explicit, force)
def self_placement(self, sections, target, flags, explicit=True, force=False, mutable=False):
placement = Placement(self, sections, target, flags, explicit, force=force, mutable=mutable)
self.placements[sections] = placement
return placement
def child_placement(self, entity, sections, target, flags, sections_db):
def child_placement(self, entity, sections, target, flags, mutable, sections_db):
child = self.add_child(entity)
child.insert(entity, sections, target, flags, sections_db)
child.insert(entity, sections, target, flags, mutable, sections_db)
def insert(self, entity, sections, target, flags, sections_db):
def insert(self, entity, sections, target, flags, mutable, sections_db):
if self.entity.specificity == entity.specificity:
# Since specificities match, create the placement in this node.
self.self_placement(sections, target, flags)
self.self_placement(sections, target, flags, mutable=mutable)
else:
# If not, create a child node and try to create the placement there.
self.child_placement(entity, sections, target, flags, sections_db)
self.child_placement(entity, sections, target, flags, mutable, sections_db)
def get_output_sections(self):
return sorted(self.placements.keys(), key=lambda x: sorted(x)) # pylint: disable=W0108
@@ -298,9 +308,9 @@ class ObjectNode(EntityNode):
self.entity = Entity(self.parent.name, self.name)
self.subplacements = list()
def child_placement(self, entity, sections, target, flags, sections_db):
def child_placement(self, entity, sections, target, flags, mutable, sections_db):
child = self.add_child(entity)
sym_placement = Placement(child, sections, target, flags, True, dryrun=True)
sym_placement = Placement(child, sections, target, flags, True, dryrun=True, mutable=mutable)
# The basis placement for sym_placement can either be
# an existing placement on this node, or nonexistent.
@@ -331,7 +341,7 @@ class ObjectNode(EntityNode):
obj_placement = self.placements[sections]
except KeyError:
# Create intermediate placement.
obj_placement = self.self_placement(sections, sym_placement.basis.target, None, False)
obj_placement = self.self_placement(sections, sym_placement.basis.target, None, False, mutable=mutable)
if obj_placement.basis.flags:
subplace = True
@@ -383,9 +393,9 @@ class Generation:
"""
# Processed mapping, scheme and section entries
EntityMapping = namedtuple('EntityMapping', 'entity sections_group target flags')
EntityMapping = namedtuple('EntityMapping', 'entity sections_group target flags mutable')
def __init__(self, check_mappings=False, check_mapping_exceptions=None):
def __init__(self, check_mappings=False, check_mapping_exceptions=None, mutable_libs=None, debug=False):
self.schemes = {}
self.placements = {}
self.mappings = {}
@@ -397,6 +407,11 @@ class Generation:
else:
self.check_mapping_exceptions = []
if mutable_libs is None:
self.mutable_libs = []
else:
self.mutable_libs = mutable_libs
def _prepare_scheme_dictionary(self):
scheme_dictionary = collections.defaultdict(dict)
@@ -442,12 +457,51 @@ class Generation:
return scheme_dictionary
def _get_section_strs(self, section):
s_list = [Sections.get_section_data_from_entry(s) for s in section.entries]
return frozenset([item for sublist in s_list for item in sublist])
def _prepare_mutable_entity_mappings(self, entity_mappings, scheme_dictionary, entities):
# Go through all mutable libraries, if any, and add entity mapping for
# each "default" section set for which the entity mapping does not yet
# exist. This adds mapping for every mutable library as if specified in
# the mapping fragment. All newly added or existing entity mappings
# related to mutable libraries are marked as mutable. This flag is used
# when the placement for it is created in the entity tree, ensuring the
# placement is forced to be emitted in the linker script and not
# considered non-significant.
for archive in self.mutable_libs:
entity = Entity(archive, '*')
for target, sections in scheme_dictionary['default'].items():
for section in sections:
sections_str = self._get_section_strs(section)
key = (entity, sections_str)
if entity_mappings.get(key):
# Mutable library already has entity mapping.
continue
entity_mappings[key] = Generation.EntityMapping(entity,
sections_str,
target,
[],
True)
for key, mapping in entity_mappings.items():
(entity, sections, target, flags, mutable) = mapping
if entity.archive not in self.mutable_libs or mutable:
# The entity either does not belong to a mutable library or is
# an archive entity that has already been marked as mutable.
continue
# Set entity as mutable.
entity_mappings[key] = Generation.EntityMapping(entity,
sections,
target,
flags,
True)
def _prepare_entity_mappings(self, scheme_dictionary, entities):
# Prepare entity mappings processed from mapping fragment entries.
def get_section_strs(section):
s_list = [Sections.get_section_data_from_entry(s) for s in section.entries]
return frozenset([item for sublist in s_list for item in sublist])
entity_mappings = dict()
for mapping in self.mappings.values():
@@ -488,7 +542,7 @@ class Generation:
if (flag.section, flag.target) == (section.name, target):
_flags.extend(flag.flags)
sections_str = get_section_strs(section)
sections_str = self._get_section_strs(section)
key = (entity, sections_str)
@@ -498,7 +552,11 @@ class Generation:
existing = None
if not existing:
entity_mappings[key] = Generation.EntityMapping(entity, sections_str, target, _flags)
entity_mappings[key] = Generation.EntityMapping(entity,
sections_str,
target,
_flags,
False)
else:
# Check for conflicts.
if target != existing.target:
@@ -511,12 +569,18 @@ class Generation:
_flags.extend(existing.flags)
entity_mappings[key] = Generation.EntityMapping(entity,
sections_str,
target, _flags)
target,
_flags,
False)
elif _flags == existing.flags:
pass
else:
raise GenerationException('Conflicting flags specified.', mapping)
# Add new mappings for mutable libraries and modify existing ones if
# they pertain to them.
self._prepare_mutable_entity_mappings(entity_mappings, scheme_dictionary, entities)
# Sort the mappings by specificity, so as to simplify
# insertion logic.
res = list(entity_mappings.values())
@@ -528,9 +592,9 @@ class Generation:
entity_mappings = self._prepare_entity_mappings(scheme_dictionary, entities)
root_node = RootNode()
for mapping in entity_mappings:
(entity, sections, target, flags) = mapping
(entity, sections, target, flags, mutable) = mapping
try:
root_node.insert(entity, sections, target, flags, entities)
root_node.insert(entity, sections, target, flags, mutable, entities)
except ValueError as e:
raise GenerationException(str(e))
tools/ldgen/ldgen/linker_script.py
+8 -2
View File
@@ -24,6 +24,7 @@ class LinkerScript:
MappingMarker = collections.namedtuple('MappingMarker', 'target indent rules')
ArraysMarker = collections.namedtuple('ArraysMarker', 'target indent rules')
MutableMarker = collections.namedtuple('MutableMarker', 'target indent rules')
def __init__(self, template_file):
self.members = []
@@ -37,16 +38,19 @@ class LinkerScript:
target = Fragment.IDENTIFIER
pattern_mapping = White(' \t') + Suppress('mapping') + Suppress('[') + target + Suppress(']')
pattern_arrays = White(' \t') + Suppress('arrays') + Suppress('[') + target + Suppress(']')
pattern_mutable = White(' \t') + Suppress('mutable') + Suppress('[') + target + Suppress(']')
# Find the markers in the template file line by line. If line does not match marker grammar,
# set it as a literal to be copied as is to the output file.
for line in lines:
parsed = False
for pattern in (pattern_arrays, pattern_mapping):
for pattern in (pattern_arrays, pattern_mapping, pattern_mutable):
try:
indent, target = pattern.parse_string(line)
if pattern is pattern_arrays:
marker = LinkerScript.ArraysMarker(target, indent, [])
elif pattern is pattern_mutable:
marker = LinkerScript.MutableMarker(target, indent, [])
else:
marker = LinkerScript.MappingMarker(target, indent, [])
self.members.append(marker)
@@ -65,8 +69,10 @@ class LinkerScript:
if isinstance(member, self.ArraysMarker):
rules = [x for x in mapping_rules[target] if x.tied]
elif isinstance(member, self.MutableMarker):
rules = [x for x in mapping_rules[target] if x.mutable and not x.tied]
else:
rules = [x for x in mapping_rules[target] if not x.tied]
rules = [x for x in mapping_rules[target] if not x.tied and not x.mutable]
member.rules.extend(rules)
except KeyError:
message = GenerationException.UNDEFINED_REFERENCE + " to target '" + target + "'."
tools/ldgen/ldgen/output_commands.py
+6 -3
View File
@@ -19,9 +19,10 @@ class AlignAtAddress:
command to be emitted.
"""
def __init__(self, alignment, tied=False):
def __init__(self, alignment, tied=False, mutable=False):
self.alignment = alignment
self.tied = tied
self.mutable = mutable
def __str__(self):
return ('. = ALIGN(%d);' % self.alignment)
@@ -43,9 +44,10 @@ class SymbolAtAddress:
an InputSectionDesc.
"""
def __init__(self, symbol, tied=False):
def __init__(self, symbol, tied=False, mutable=False):
self.symbol = symbol
self.tied = tied
self.mutable = mutable
def __str__(self):
return ('%s = ABSOLUTE(.);' % self.symbol)
@@ -65,7 +67,7 @@ class InputSectionDesc:
the emitted input section description.
"""
def __init__(self, entity, sections, exclusions=None, keep=False, sort=None, tied=False):
def __init__(self, entity, sections, exclusions=None, keep=False, sort=None, tied=False, mutable=False):
assert entity.specificity != Entity.Specificity.SYMBOL
self.entity = entity
@@ -83,6 +85,7 @@ class InputSectionDesc:
self.keep = keep
self.sort = sort
self.tied = tied
self.mutable = mutable
def __str__(self):
sections_string = ''