esp-idf/tools/ldgen/generation.py

#
# Copyright 2021 Espressif Systems (Shanghai) CO LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import collections
import fnmatch
import itertools
from collections import namedtuple

from entity import Entity
from fragments import Mapping, Scheme, Sections
from ldgen_common import LdGenFailure
from output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress


class Placement():

    def __init__(self, node, sections, target, flags, explicit, force=False, dryrun=False):
        self.node = node
        self.sections = sections
        self.target = target
        self.flags = flags

        self.exclusions = set()
        self.subplacements = set()

        # Force this placement to be output
        self.force = force

        # This placement was created from a mapping
        # fragment entry.
        self.explicit = explicit

        # Find basis placement. A basis placement is a placement
        # on the parent (or parent's parent and so on and so forth)
        # that operates on the same section as this one.
        parent = node.parent
        candidate = None
        while parent:
            try:
                candidate = parent.placements[sections]
            except KeyError:
                pass

            if candidate and candidate.is_significant():
                break
            else:
                parent = parent.parent

        self.basis = candidate

        if self.is_significant() and not dryrun and self.basis:
            self.basis.add_exclusion(self)

    def is_significant(self):
        # Check if the placement is significant. Significant placements
        # are the end of a basis chain (not self.basis) or a change
        # in target (self.target != self.basis.target)
        #
        # Placement can also be a basis if it has flags
        # (self.flags) or its basis has flags (self.basis.flags)
        return (not self.basis or
                self.target != self.basis.target or
                (self.flags and not self.basis.flags) or
                (not self.flags and self.basis.flags) or
                self.force)

    def force_significant(self):
        if not self.is_significant():
            self.force = True
            if self.basis:
                self.basis.add_exclusion(self)

    def add_exclusion(self, exclusion):
        self.exclusions.add(exclusion)

    def add_subplacement(self, subplacement):
        self.subplacements.add(subplacement)


class EntityNode():

    def __init__(self, parent, name):
        self.children = []
        self.parent = parent
        self.name = name
        self.child_t = EntityNode
        self.entity = None
        self.placements = dict()

    def add_child(self, entity):
        child_specificity = self.entity.specificity.value + 1
        assert(child_specificity <= Entity.Specificity.SYMBOL.value)
        name = entity[Entity.Specificity(child_specificity)]
        assert(name and name != Entity.ALL)

        child = [c for c in self.children if c.name == name]
        assert(len(child) <= 1)

        if not child:
            child = self.child_t(self, name)
            self.children.append(child)
        else:
            child = child[0]

        return child

    def get_output_commands(self):
        commands = collections.defaultdict(list)

        def process_commands(cmds):
            for (target, commands_list) in cmds.items():
                commands[target].extend(commands_list)

        # Process the commands generated from this node
        node_commands = self.get_node_output_commands()
        process_commands(node_commands)

        # Process the commands generated from this node's children
        # recursively
        for child in sorted(self.children, key=lambda c: c.name):
            children_commands = child.get_output_commands()
            process_commands(children_commands)

        return commands

    def get_node_output_commands(self):
        commands = collections.defaultdict(list)

        for sections in self.get_output_sections():
            placement = self.placements[sections]
            if placement.is_significant():
                assert(placement.node == self)

                keep = False
                sort = None
                surround = []

                placement_flags = placement.flags if placement.flags is not None else []

                for flag in placement_flags:
                    if isinstance(flag, Mapping.Keep):
                        keep = True
                    elif isinstance(flag, Mapping.Sort):
                        sort = (flag.first, flag.second)
                    else:  # emit or align
                        surround.append(flag)

                for flag in surround:
                    if flag.pre:
                        if isinstance(flag, Mapping.Emit):
                            commands[placement.target].append(SymbolAtAddress('_%s_start' % flag.symbol))
                        else:  # align
                            commands[placement.target].append(AlignAtAddress(flag.alignment))

                # This is for expanded object node and symbol node placements without checking for
                # the type.
                placement_sections = frozenset(placement.sections)
                command_sections = sections if sections == placement_sections else placement_sections

                command = InputSectionDesc(placement.node.entity, command_sections, [e.node.entity for e in placement.exclusions], keep, sort)
                commands[placement.target].append(command)

                # Generate commands for intermediate, non-explicit exclusion placements here, so that they can be enclosed by
                # flags that affect the parent placement.
                for subplacement in placement.subplacements:
                    if not subplacement.flags and not subplacement.explicit:
                        command = InputSectionDesc(subplacement.node.entity, subplacement.sections,
                                                   [e.node.entity for e in subplacement.exclusions], keep, sort)
                        commands[placement.target].append(command)

                for flag in surround:
                    if flag.post:
                        if isinstance(flag, Mapping.Emit):
                            commands[placement.target].append(SymbolAtAddress('_%s_end' % flag.symbol))
                        else:  # align
                            commands[placement.target].append(AlignAtAddress(flag.alignment))

        return commands

    def self_placement(self, sections, target, flags, explicit=True, force=False):
        placement = Placement(self, sections, target, flags, explicit, force)
        self.placements[sections] = placement
        return placement

    def child_placement(self, entity, sections, target, flags, sections_db):
        child = self.add_child(entity)
        child.insert(entity, sections, target, flags, sections_db)

    def insert(self, entity, sections, target, flags, sections_db):
        if self.entity.specificity == entity.specificity:
            # Since specificities match, create the placement in this node.
            self.self_placement(sections, target, flags)
        else:
            # If not, create a child node and try to create the placement there.
            self.child_placement(entity, sections, target, flags, sections_db)

    def get_output_sections(self):
        return sorted(self.placements.keys(), key=' '.join)


class SymbolNode(EntityNode):

    def __init__(self, parent, name):
        EntityNode.__init__(self, parent, name)
        self.entity = Entity(self.parent.parent.name, self.parent.name)


class ObjectNode(EntityNode):

    def __init__(self, parent, name):
        EntityNode.__init__(self, parent, name)
        self.child_t = SymbolNode
        self.entity = Entity(self.parent.name, self.name)
        self.subplacements = list()

    def child_placement(self, entity, sections, target, flags, sections_db):
        child = self.add_child(entity)
        sym_placement = Placement(child, sections, target, flags, True, dryrun=True)

        # The basis placement for sym_placement can either be
        # an existing placement on this node, or nonexistent.
        if sym_placement.is_significant():
            try:
                obj_sections = self.placements[sections].sections
            except KeyError:
                obj_sections = None

            if not obj_sections or obj_sections == sections:
                # Expand this section for the first time
                found_sections = sections_db.get_sections(self.parent.name, self.name)
                obj_sections = []
                for s in sections:
                    obj_sections.extend(fnmatch.filter(found_sections, s))

            if obj_sections:
                symbol = entity.symbol
                remove_sections = [s.replace('.*', '.%s' % symbol) for s in sections if '.*' in s]
                filtered_sections = [s for s in obj_sections if s not in remove_sections]

                if set(filtered_sections) != set(obj_sections):
                    if sym_placement.basis:
                        subplace = False
                        try:
                            # If existing placement exists, make sure that
                            # it is emitted.
                            obj_placement = self.placements[sections]
                        except KeyError:
                            # Create intermediate placement.
                            obj_placement = self.self_placement(sections, sym_placement.basis.target, None, False)
                            if obj_placement.basis.flags:
                                subplace = True

                        if subplace:
                            obj_placement.basis.add_subplacement(obj_placement)
                            self.subplacements.append(sections)
                        else:
                            obj_placement.force_significant()

                        obj_placement.sections = filtered_sections
                        sym_placement.basis = obj_placement

                    sym_placement.sections = remove_sections
                    child.placements[sections] = sym_placement

    def get_output_sections(self):
        output_sections = [key for key in self.placements if key not in self.subplacements]
        return sorted(output_sections, key=' '.join)


class ArchiveNode(EntityNode):

    def __init__(self, parent, name):
        EntityNode.__init__(self, parent, name)
        self.child_t = ObjectNode
        self.entity = Entity(self.name)


class RootNode(EntityNode):
    def __init__(self):
        EntityNode.__init__(self, None, Entity.ALL)
        self.child_t = ArchiveNode
        self.entity = Entity('*')


class Generation:
    """
    Implements generation of placement based on collected sections, scheme and mapping fragment.
    """

    # Processed mapping, scheme and section entries
    EntityMapping = namedtuple('EntityMapping', 'entity sections_group target flags')

    def __init__(self, check_mappings=False, check_mapping_exceptions=None):
        self.schemes = {}
        self.placements = {}
        self.mappings = {}

        self.check_mappings = check_mappings

        if check_mapping_exceptions:
            self.check_mapping_exceptions = check_mapping_exceptions
        else:
            self.check_mapping_exceptions = []

    def _prepare_scheme_dictionary(self):
        scheme_dictionary = collections.defaultdict(dict)

        # Collect sections into buckets based on target name
        for scheme in self.schemes.values():
            sections_bucket = collections.defaultdict(list)

            for (sections_name, target_name) in scheme.entries:
                # Get the sections under the bucket 'target_name'. If this bucket does not exist
                # is is created automatically
                sections_in_bucket = sections_bucket[target_name]

                try:
                    sections = self.placements[sections_name]
                except KeyError:
                    message = GenerationException.UNDEFINED_REFERENCE + " to sections '" + sections_name + "'."
                    raise GenerationException(message, scheme)

                sections_in_bucket.append(sections)

            scheme_dictionary[scheme.name] = sections_bucket

        # Search for and raise exception on first instance of sections mapped to multiple targets
        for (scheme_name, sections_bucket) in scheme_dictionary.items():
            for sections_a, sections_b in itertools.combinations(sections_bucket.values(), 2):
                set_a = set()
                set_b = set()

                for sections in sections_a:
                    set_a.update(sections.entries)

                for sections in sections_b:
                    set_b.update(sections.entries)

                intersection = set_a.intersection(set_b)

                # If the intersection is a non-empty set, it means sections are mapped to multiple
                # targets. Raise exception.
                if intersection:
                    scheme = self.schemes[scheme_name]
                    message = 'Sections ' + str(intersection) + ' mapped to multiple targets.'
                    raise GenerationException(message, scheme)

        return scheme_dictionary

    def _prepare_entity_mappings(self, scheme_dictionary, entities):
        # Prepare entity mappings processed from mapping fragment entries.
        def get_section_strs(section):
            s_list = [Sections.get_section_data_from_entry(s) for s in section.entries]
            return frozenset([item for sublist in s_list for item in sublist])

        entity_mappings = dict()

        for mapping in self.mappings.values():
            archive = mapping.archive

            for (obj, symbol, scheme_name) in mapping.entries:
                entity = Entity(archive, obj, symbol)

                # Check the entity exists
                if (self.check_mappings and
                        entity.specificity.value > Entity.Specificity.ARCHIVE.value and
                        mapping.name not in self.check_mapping_exceptions):
                    if not entities.check_exists(entity):
                        message = "'%s' not found" % str(entity)
                        raise GenerationException(message, mapping)

                if (obj, symbol, scheme_name) in mapping.flags.keys():
                    flags = mapping.flags[(obj, symbol, scheme_name)]
                    # Check if all section->target defined in the current
                    # scheme.
                    for (s, t, f) in flags:
                        if (t not in scheme_dictionary[scheme_name].keys() or
                                s not in [_s.name for _s in scheme_dictionary[scheme_name][t]]):

                            message = "%s->%s not defined in scheme '%s'" % (s, t, scheme_name)
                            raise GenerationException(message, mapping)
                else:
                    flags = None

                # Create placement for each 'section -> target' in the scheme.
                for (target, sections) in scheme_dictionary[scheme_name].items():
                    for section in sections:
                        # Find the applicable flags
                        _flags = []

                        if flags:
                            for (s, t, f) in flags:
                                if (s, t) == (section.name, target):
                                    _flags.extend(f)

                        sections_str = get_section_strs(section)

                        key = (entity, section.name)

                        try:
                            existing = entity_mappings[key]
                        except KeyError:
                            existing = None

                        if not existing:
                            entity_mappings[key] = Generation.EntityMapping(entity, sections_str, target, _flags)
                        else:
                            # Check for conflicts.
                            if (target != existing.target):
                                raise GenerationException('Sections mapped to multiple targets.', mapping)

                            # Combine flags here if applicable, to simplify
                            # insertion logic.
                            if (_flags or existing.flags):
                                if ((_flags and not existing.flags) or (not _flags and existing.flags)):
                                    _flags.extend(existing.flags)
                                    entity_mappings[key] = Generation.EntityMapping(entity,
                                                                                    sections_str,
                                                                                    target, _flags)
                                elif (_flags == existing.flags):
                                    pass
                                else:
                                    raise GenerationException('Conflicting flags specified.', mapping)

        # Sort the mappings by specificity, so as to simplify
        # insertion logic.
        res = list(entity_mappings.values())
        res.sort(key=lambda m: m.entity)
        return res

    def generate(self, entities):
        scheme_dictionary = self._prepare_scheme_dictionary()
        entity_mappings = self._prepare_entity_mappings(scheme_dictionary, entities)
        root_node = RootNode()
        for mapping in entity_mappings:
            (entity, sections, target, flags) = mapping
            try:
                root_node.insert(entity, sections, target, flags, entities)
            except ValueError as e:
                raise GenerationException(str(e))

        # Traverse the tree, creating the placements
        commands = root_node.get_output_commands()

        return commands

    def add_fragments_from_file(self, fragment_file):
        for fragment in fragment_file.fragments:
            dict_to_append_to = None

            if isinstance(fragment, Mapping) and fragment.deprecated and fragment.name in self.mappings.keys():
                self.mappings[fragment.name].entries |= fragment.entries
            else:
                if isinstance(fragment, Scheme):
                    dict_to_append_to = self.schemes
                elif isinstance(fragment, Sections):
                    dict_to_append_to = self.placements
                else:
                    dict_to_append_to = self.mappings

                # Raise exception when the fragment of the same type is already in the stored fragments
                if fragment.name in dict_to_append_to.keys():
                    stored = dict_to_append_to[fragment.name].path
                    new = fragment.path
                    message = "Duplicate definition of fragment '%s' found in %s and %s." % (fragment.name, stored, new)
                    raise GenerationException(message)

                dict_to_append_to[fragment.name] = fragment


class GenerationException(LdGenFailure):
    """
    Exception for linker script generation failures such as undefined references/ failure to
    evaluate conditions, duplicate mappings, etc.
    """

    UNDEFINED_REFERENCE = 'Undefined reference'

    def __init__(self, message, fragment=None):
        self.fragment = fragment
        self.message = message

    def __str__(self):
        if self.fragment:
            return "%s\nIn fragment '%s' defined in '%s'." % (self.message, self.fragment.name, self.fragment.path)
        else:
            return self.message