Skip to content

Commit

Permalink
arm64: per-func __patchable_function_entries sections
Browse files Browse the repository at this point in the history 
new clang toolchain on arm64 produces individual __patchable_function_entries
sections for each patchable func, in -ffunction-sections mode,
rather than traditional single __mcount_loc section.

Bend the existing logic to detect this multiplicity in the incoming
kelf objects, and allocate N identical one-entry sections.
These are retrieved as needed by a new function: find_nth_section_by_name()
and attached to the .text sections they describe.

These __pfe section are not actually arm64-specific, but a generic
enhancement across gcc & clang, to allow better garbage collection of
unreferenced object sections, and mcount/pfe objects which refer to them.
The __pfe sections are combined in kernel-or-module final link,
from 5.19.9's 9440155ccb948f8e3ce5308907a2e7378799be60.
From clang-11, __pfe is supported for x86, though not yet used by kernel

The split between allocate/populate phases here is necessary to
enumerate/populate the outgoing section-headers before beginning to
produce output sections

Also adds some missing \n to log_debug()s

Signed-off-by: Pete Swain <[email protected]>
  • Loading branch information
@swine
swine committed Jan 9, 2023
1 parent 6492900 commit f1b8037
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 36 deletions.
120 changes: 91 additions & 29 deletions kpatch-build/create-diff-object.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ enum subsection {
enum loglevel loglevel = NORMAL;

bool KLP_ARCH;
bool multi_pfe;

int jump_label_errors, static_call_errors;

Expand Down Expand Up @@ -3231,7 +3232,7 @@ static void kpatch_create_patches_sections(struct kpatch_elf *kelf,
if (sym->bind == STB_LOCAL && symbol.global)
ERROR("can't find local symbol '%s' in symbol table", sym->name);

log_debug("lookup for %s: obj=%s sympos=%lu size=%lu",
log_debug("lookup for %s: obj=%s sympos=%lu size=%lu\n",
sym->name, symbol.objname, symbol.sympos,
symbol.size);

Expand Down Expand Up @@ -3603,7 +3604,7 @@ static void kpatch_create_intermediate_sections(struct kpatch_elf *kelf,
ERROR("can't find symbol '%s' in symbol table",
rela->sym->name);

log_debug("lookup for %s: obj=%s sympos=%lu",
log_debug("lookup for %s: obj=%s sympos=%lu\n",
rela->sym->name, symbol.objname,
symbol.sympos);

Expand Down Expand Up @@ -3757,29 +3758,51 @@ static void kpatch_alloc_mcount_sections(struct kpatch_elf *kelf, struct kpatch_
{
int nr;
struct symbol *sym;
int text_idx = 0;

nr = 0;
list_for_each_entry(sym, &kelfout->symbols, list)
list_for_each_entry(sym, &kelfout->symbols, list) {
if (sym->type == STT_FUNC && sym->status != SAME &&
sym->has_func_profiling)
sym->has_func_profiling) {
text_idx = sym->sec->index;
nr++;
}
}

/* create text/rela section pair */
switch(kelf->arch) {
case AARCH64: {
struct section *sec, *tmp;

sec = create_section_pair(kelfout, "__patchable_function_entries", sizeof(void *), nr);
struct section *sec;
int entries = multi_pfe ? 1 : nr;
int copies = multi_pfe ? nr : 1;
int flags = 0, rflags = 0;

/*
* Depending on the compiler the __patchable_function_entries section
* can be ordered or not, copy this flag to the section we created to
* avoid:
* ld: __patchable_function_entries has both ordered [...] and unordered [...] sections
*/
tmp = find_section_by_name(&kelf->sections, "__patchable_function_entries");
sec->sh.sh_flags |= (tmp->sh.sh_flags & SHF_LINK_ORDER);
sec->sh.sh_link = 1;
sec = find_section_by_name(&kelf->sections, "__patchable_function_entries");
if (sec) {
flags = (sec->sh.sh_flags & (SHF_LINK_ORDER|SHF_WRITE));
if (sec->rela)
rflags = (sec->rela->sh.sh_flags & (SHF_LINK_ORDER|SHF_WRITE));
}

for (nr = 0; nr < copies; nr++) {
sec = create_section_pair(kelfout,
"__patchable_function_entries",
sizeof(void *), entries);

sec->sh.sh_flags |= flags;
if (sec->rela)
sec->rela->sh.sh_flags |= rflags;
if (multi_pfe)
sec->sh.sh_link = 0;
else
sec->sh.sh_link = text_idx;
}
break;
}
case PPC64:
Expand Down Expand Up @@ -3808,11 +3831,14 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf)
struct symbol *sym;
struct rela *rela, *mcount_rela;
void **funcs;
unsigned long insn_offset = 0;

switch(kelf->arch) {
case AARCH64:
sec = find_section_by_name(&kelf->sections, "__patchable_function_entries");
if (multi_pfe)
sec = NULL;
else
sec = find_section_by_name(&kelf->sections,
"__patchable_function_entries");
break;
case PPC64:
case X86_64:
Expand All @@ -3822,12 +3848,20 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf)
default:
ERROR("unsupported arch\n");
}
relasec = sec->rela;
nr = (int) (sec->data->d_size / sizeof(void *));

if (multi_pfe) {
relasec = NULL;
nr = 0;
} else {
relasec = sec->rela;
nr = (int) (sec->data->d_size / sizeof(void *));
}

/* populate sections */
index = 0;
list_for_each_entry(sym, &kelf->symbols, list) {
unsigned long insn_offset = 0;

if (sym->type != STT_FUNC || sym->status == SAME)
continue;

Expand All @@ -3843,7 +3877,6 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf)
int i;

insn = sym->sec->data->d_buf;
insn_offset = 0;

/*
* If BTI (Branch Target Identification) is enabled then there
Expand Down Expand Up @@ -3934,6 +3967,18 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf)
ERROR("unsupported arch");
}

if (multi_pfe) {
sec = find_nth_section_by_name(&kelf->sections, nr, "__patchable_function_entries");
if (!sec)
ERROR("cannot retrieve pre-allocated __pfe #%d\n", nr);

relasec = sec->rela;
sym->sec->pfe = sec;
sec->sh.sh_link = sec->index;

nr++;
}

/*
* 'rela' points to the mcount/fentry call.
*
Expand All @@ -3943,7 +3988,13 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf)
mcount_rela->sym = sym;
mcount_rela->type = absolute_rela_type(kelf);
mcount_rela->addend = insn_offset - sym->sym.st_value;
mcount_rela->offset = (unsigned int) (index * sizeof(*funcs));

if (multi_pfe) {
mcount_rela->offset = 0;
sec = NULL;
} else {
mcount_rela->offset = (unsigned int) (index * sizeof(*funcs));
}

index++;
}
Expand Down Expand Up @@ -4096,27 +4147,30 @@ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf)
struct symbol *sym;
struct rela *rela;
unsigned char *insn;

list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->type != STT_FUNC || !sym->sec || !sym->sec->rela)
continue;

switch(kelf->arch) {
case AARCH64: {
struct section *sec = find_section_by_name(&kelf->sections,
"__patchable_function_entries");
/*
* If we can't find the __patchable_function_entries section or
* there are no relocations in it then not patchable.
*/
if (!sec || !sec->rela)
return;
list_for_each_entry(rela, &sec->rela->relas, list) {
if (rela->sym->sec && sym->sec == rela->sym->sec) {
sym->has_func_profiling = 1;
break;
struct section *sec;

list_for_each_entry(sec, &kelf->sections, list) {
if (strcmp(sec->name, "__patchable_function_entries"))
continue;
if (multi_pfe && sym->sec->pfe != sec)
continue;
if (!sec->rela)
continue;

list_for_each_entry(rela, &sec->rela->relas, list) {
if (rela->sym->sec && sym->sec == rela->sym->sec) {
sym->has_func_profiling = 1;
goto next_symbol;
}
}
}

break;
}
case PPC64:
Expand Down Expand Up @@ -4149,6 +4203,7 @@ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf)
default:
ERROR("unsupported arch");
}
next_symbol:;
}
}

Expand Down Expand Up @@ -4196,6 +4251,12 @@ static error_t parse_opt (int key, char *arg, struct argp_state *state)
return 0;
}

static bool has_multi_pfe(struct kpatch_elf *kelf)
{
return !!find_nth_section_by_name(&kelf->sections, 1,
"__patchable_function_entries");
}

static struct argp argp = { options, parse_opt, args_doc, NULL };

int main(int argc, char *argv[])
Expand Down Expand Up @@ -4229,6 +4290,7 @@ int main(int argc, char *argv[])

kelf_orig = kpatch_elf_open(orig_obj);
kelf_patched = kpatch_elf_open(patched_obj);
multi_pfe = has_multi_pfe(kelf_orig) || has_multi_pfe(kelf_patched);
kpatch_find_func_profiling_calls(kelf_orig);
kpatch_find_func_profiling_calls(kelf_patched);

Expand Down
32 changes: 25 additions & 7 deletions kpatch-build/kpatch-elf.c
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,17 +88,29 @@ struct section *find_section_by_index(struct list_head *list, unsigned int index
return NULL;
}

struct section *find_section_by_name(struct list_head *list, const char *name)
struct section *find_nth_section_by_name( struct list_head *list, int nth, const char *name)
{
struct section *sec;

list_for_each_entry(sec, list, list)
if (!strcmp(sec->name, name))
return sec;
if (!list || !list->next || !name)
return NULL;

list_for_each_entry(sec, list, list) {
if (strcmp(sec->name, name))
continue;
if (--nth >= 0)
continue;
return sec;
}

return NULL;
}

struct section *find_section_by_name(struct list_head *list, const char *name)
{
return find_nth_section_by_name(list, 0, name);
}

struct symbol *find_symbol_by_index(struct list_head *list, size_t index)
{
struct symbol *sym;
Expand Down Expand Up @@ -881,11 +893,17 @@ void kpatch_reindex_elements(struct kpatch_elf *kelf)
index = 0;
list_for_each_entry(sym, &kelf->symbols, list) {
sym->index = index++;
if (sym->sec)
if (sym->sec) {
sym->sym.st_shndx = (unsigned short)sym->sec->index;
else if (sym->sym.st_shndx != SHN_ABS &&
sym->sym.st_shndx != SHN_LIVEPATCH)
if (sym->sec->pfe) {
sym->sec->pfe->sh.sh_link = sym->sec->index;
if (sym->sec->pfe->rela)
sym->sec->pfe->rela->sh.sh_info = sym->sec->index;
}
} else if (sym->sym.st_shndx != SHN_ABS &&
sym->sym.st_shndx != SHN_LIVEPATCH) {
sym->sym.st_shndx = SHN_UNDEF;
}
}
}

Expand Down
3 changes: 3 additions & 0 deletions kpatch-build/kpatch-elf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ struct section {
struct symbol *secsym, *sym;
};
};
struct section *pfe; /* arm64 per-func __patchable_function_entries */
};

enum symbol_strip {
Expand Down Expand Up @@ -136,6 +137,8 @@ bool is_debug_section(struct section *sec);

struct section *find_section_by_index(struct list_head *list, unsigned int index);
struct section *find_section_by_name(struct list_head *list, const char *name);
struct section *find_nth_section_by_name(struct list_head *list, int nth,
const char *name);
struct symbol *find_symbol_by_index(struct list_head *list, size_t index);
struct symbol *find_symbol_by_name(struct list_head *list, const char *name);
struct rela *find_rela_by_offset(struct section *relasec, unsigned int offset);
Expand Down

0 comments on commit f1b8037

Please sign in to comment.