diff --git a/kpatch-build/create-diff-object.c b/kpatch-build/create-diff-object.c index 2c8b04544..212e8f226 100644 --- a/kpatch-build/create-diff-object.c +++ b/kpatch-build/create-diff-object.c @@ -70,6 +70,7 @@ enum subsection { enum loglevel loglevel = NORMAL; bool KLP_ARCH; +bool multi_pfe; int jump_label_errors, static_call_errors; @@ -3231,7 +3232,7 @@ static void kpatch_create_patches_sections(struct kpatch_elf *kelf, if (sym->bind == STB_LOCAL && symbol.global) ERROR("can't find local symbol '%s' in symbol table", sym->name); - log_debug("lookup for %s: obj=%s sympos=%lu size=%lu", + log_debug("lookup for %s: obj=%s sympos=%lu size=%lu\n", sym->name, symbol.objname, symbol.sympos, symbol.size); @@ -3603,7 +3604,7 @@ static void kpatch_create_intermediate_sections(struct kpatch_elf *kelf, ERROR("can't find symbol '%s' in symbol table", rela->sym->name); - log_debug("lookup for %s: obj=%s sympos=%lu", + log_debug("lookup for %s: obj=%s sympos=%lu\n", rela->sym->name, symbol.objname, symbol.sympos); @@ -3757,19 +3758,24 @@ static void kpatch_alloc_mcount_sections(struct kpatch_elf *kelf, struct kpatch_ { int nr; struct symbol *sym; + int text_idx = 0; nr = 0; - list_for_each_entry(sym, &kelfout->symbols, list) + list_for_each_entry(sym, &kelfout->symbols, list) { if (sym->type == STT_FUNC && sym->status != SAME && - sym->has_func_profiling) + sym->has_func_profiling) { + text_idx = sym->sec->index; nr++; + } + } /* create text/rela section pair */ switch(kelf->arch) { case AARCH64: { - struct section *sec, *tmp; - - sec = create_section_pair(kelfout, "__patchable_function_entries", sizeof(void *), nr); + struct section *sec; + int entries = multi_pfe ? 1 : nr; + int copies = multi_pfe ? nr : 1; + int flags = 0, rflags = 0; /* * Depending on the compiler the __patchable_function_entries section @@ -3777,9 +3783,26 @@ static void kpatch_alloc_mcount_sections(struct kpatch_elf *kelf, struct kpatch_ * avoid: * ld: __patchable_function_entries has both ordered [...] and unordered [...] sections */ - tmp = find_section_by_name(&kelf->sections, "__patchable_function_entries"); - sec->sh.sh_flags |= (tmp->sh.sh_flags & SHF_LINK_ORDER); - sec->sh.sh_link = 1; + sec = find_section_by_name(&kelf->sections, "__patchable_function_entries"); + if (sec) { + flags = (sec->sh.sh_flags & (SHF_LINK_ORDER|SHF_WRITE)); + if (sec->rela) + rflags = (sec->rela->sh.sh_flags & (SHF_LINK_ORDER|SHF_WRITE)); + } + + for (nr = 0; nr < copies; nr++) { + sec = create_section_pair(kelfout, + "__patchable_function_entries", + sizeof(void *), entries); + + sec->sh.sh_flags |= flags; + if (sec->rela) + sec->rela->sh.sh_flags |= rflags; + if (multi_pfe) + sec->sh.sh_link = 0; + else + sec->sh.sh_link = text_idx; + } break; } case PPC64: @@ -3808,11 +3831,14 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf) struct symbol *sym; struct rela *rela, *mcount_rela; void **funcs; - unsigned long insn_offset = 0; switch(kelf->arch) { case AARCH64: - sec = find_section_by_name(&kelf->sections, "__patchable_function_entries"); + if (multi_pfe) + sec = NULL; + else + sec = find_section_by_name(&kelf->sections, + "__patchable_function_entries"); break; case PPC64: case X86_64: @@ -3822,12 +3848,20 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf) default: ERROR("unsupported arch\n"); } - relasec = sec->rela; - nr = (int) (sec->data->d_size / sizeof(void *)); + + if (multi_pfe) { + relasec = NULL; + nr = 0; + } else { + relasec = sec->rela; + nr = (int) (sec->data->d_size / sizeof(void *)); + } /* populate sections */ index = 0; list_for_each_entry(sym, &kelf->symbols, list) { + unsigned long insn_offset = 0; + if (sym->type != STT_FUNC || sym->status == SAME) continue; @@ -3843,7 +3877,6 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf) int i; insn = sym->sec->data->d_buf; - insn_offset = 0; /* * If BTI (Branch Target Identification) is enabled then there @@ -3934,6 +3967,18 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf) ERROR("unsupported arch"); } + if (multi_pfe) { + sec = find_nth_section_by_name(&kelf->sections, nr, "__patchable_function_entries"); + if (!sec) + ERROR("cannot retrieve pre-allocated __pfe #%d\n", nr); + + relasec = sec->rela; + sym->sec->pfe = sec; + sec->sh.sh_link = sec->index; + + nr++; + } + /* * 'rela' points to the mcount/fentry call. * @@ -3943,7 +3988,13 @@ static void kpatch_populate_mcount_sections(struct kpatch_elf *kelf) mcount_rela->sym = sym; mcount_rela->type = absolute_rela_type(kelf); mcount_rela->addend = insn_offset - sym->sym.st_value; - mcount_rela->offset = (unsigned int) (index * sizeof(*funcs)); + + if (multi_pfe) { + mcount_rela->offset = 0; + sec = NULL; + } else { + mcount_rela->offset = (unsigned int) (index * sizeof(*funcs)); + } index++; } @@ -4096,27 +4147,30 @@ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf) struct symbol *sym; struct rela *rela; unsigned char *insn; + list_for_each_entry(sym, &kelf->symbols, list) { if (sym->type != STT_FUNC || !sym->sec || !sym->sec->rela) continue; switch(kelf->arch) { case AARCH64: { - struct section *sec = find_section_by_name(&kelf->sections, - "__patchable_function_entries"); - /* - * If we can't find the __patchable_function_entries section or - * there are no relocations in it then not patchable. - */ - if (!sec || !sec->rela) - return; - list_for_each_entry(rela, &sec->rela->relas, list) { - if (rela->sym->sec && sym->sec == rela->sym->sec) { - sym->has_func_profiling = 1; - break; + struct section *sec; + + list_for_each_entry(sec, &kelf->sections, list) { + if (strcmp(sec->name, "__patchable_function_entries")) + continue; + if (multi_pfe && sym->sec->pfe != sec) + continue; + if (!sec->rela) + continue; + + list_for_each_entry(rela, &sec->rela->relas, list) { + if (rela->sym->sec && sym->sec == rela->sym->sec) { + sym->has_func_profiling = 1; + goto next_symbol; + } } } - break; } case PPC64: @@ -4149,6 +4203,7 @@ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf) default: ERROR("unsupported arch"); } + next_symbol:; } } @@ -4196,6 +4251,12 @@ static error_t parse_opt (int key, char *arg, struct argp_state *state) return 0; } +static bool has_multi_pfe(struct kpatch_elf *kelf) +{ + return !!find_nth_section_by_name(&kelf->sections, 1, + "__patchable_function_entries"); +} + static struct argp argp = { options, parse_opt, args_doc, NULL }; int main(int argc, char *argv[]) @@ -4229,6 +4290,7 @@ int main(int argc, char *argv[]) kelf_orig = kpatch_elf_open(orig_obj); kelf_patched = kpatch_elf_open(patched_obj); + multi_pfe = has_multi_pfe(kelf_orig) || has_multi_pfe(kelf_patched); kpatch_find_func_profiling_calls(kelf_orig); kpatch_find_func_profiling_calls(kelf_patched); diff --git a/kpatch-build/kpatch-elf.c b/kpatch-build/kpatch-elf.c old mode 100644 new mode 100755 index 4c31bac3f..3f7197452 --- a/kpatch-build/kpatch-elf.c +++ b/kpatch-build/kpatch-elf.c @@ -88,17 +88,29 @@ struct section *find_section_by_index(struct list_head *list, unsigned int index return NULL; } -struct section *find_section_by_name(struct list_head *list, const char *name) +struct section *find_nth_section_by_name( struct list_head *list, int nth, const char *name) { struct section *sec; - list_for_each_entry(sec, list, list) - if (!strcmp(sec->name, name)) - return sec; + if (!list || !list->next || !name) + return NULL; + + list_for_each_entry(sec, list, list) { + if (strcmp(sec->name, name)) + continue; + if (--nth >= 0) + continue; + return sec; + } return NULL; } +struct section *find_section_by_name(struct list_head *list, const char *name) +{ + return find_nth_section_by_name(list, 0, name); +} + struct symbol *find_symbol_by_index(struct list_head *list, size_t index) { struct symbol *sym; @@ -881,11 +893,17 @@ void kpatch_reindex_elements(struct kpatch_elf *kelf) index = 0; list_for_each_entry(sym, &kelf->symbols, list) { sym->index = index++; - if (sym->sec) + if (sym->sec) { sym->sym.st_shndx = (unsigned short)sym->sec->index; - else if (sym->sym.st_shndx != SHN_ABS && - sym->sym.st_shndx != SHN_LIVEPATCH) + if (sym->sec->pfe) { + sym->sec->pfe->sh.sh_link = sym->sec->index; + if (sym->sec->pfe->rela) + sym->sec->pfe->rela->sh.sh_info = sym->sec->index; + } + } else if (sym->sym.st_shndx != SHN_ABS && + sym->sym.st_shndx != SHN_LIVEPATCH) { sym->sym.st_shndx = SHN_UNDEF; + } } } diff --git a/kpatch-build/kpatch-elf.h b/kpatch-build/kpatch-elf.h index 8d06d858f..0429c99fb 100644 --- a/kpatch-build/kpatch-elf.h +++ b/kpatch-build/kpatch-elf.h @@ -65,6 +65,7 @@ struct section { struct symbol *secsym, *sym; }; }; + struct section *pfe; /* arm64 per-func __patchable_function_entries */ }; enum symbol_strip { @@ -136,6 +137,8 @@ bool is_debug_section(struct section *sec); struct section *find_section_by_index(struct list_head *list, unsigned int index); struct section *find_section_by_name(struct list_head *list, const char *name); +struct section *find_nth_section_by_name(struct list_head *list, int nth, + const char *name); struct symbol *find_symbol_by_index(struct list_head *list, size_t index); struct symbol *find_symbol_by_name(struct list_head *list, const char *name); struct rela *find_rela_by_offset(struct section *relasec, unsigned int offset);