create-diff-object.c

/*
 * create-diff-object.c
 *
 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
 * Copyright (C) 2013-2014 Josh Poimboeuf <jpoimboe@redhat.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA,
 * 02110-1301, USA.
 */

/*
 * This file contains the heart of the ELF object differencing engine.
 *
 * The tool takes two ELF objects from two versions of the same source
 * file; a "orig" object and a "patched" object.  These object need to have
 * been compiled with the -ffunction-sections and -fdata-sections GCC options.
 *
 * The tool compares the objects at a section level to determine what
 * sections have changed.  Once a list of changed sections has been generated,
 * various rules are applied to determine any object local sections that
 * are dependencies of the changed section and also need to be included in
 * the output object.
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <err.h>
#include <gelf.h>
#include <argp.h>
#include <libgen.h>
#include <unistd.h>

#include "list.h"
#include "lookup.h"
#include "kpatch-patch.h"
#include "kpatch-elf.h"
#include "kpatch-intermediate.h"
#include "kpatch.h"

#define DIFF_FATAL(format, ...) \
({ \
	fprintf(stderr, "ERROR: %s: " format "\n", childobj, ##__VA_ARGS__); \
	errx(EXIT_STATUS_DIFF_FATAL, "unreconcilable difference"); \
})

char *childobj;

enum subsection {
	SUBSECTION_NORMAL,
	SUBSECTION_HOT,
	SUBSECTION_UNLIKELY
};

enum loglevel loglevel = NORMAL;

bool KLP_ARCH;

int jump_label_errors, static_call_errors;

/*******************
 * Data structures
 * ****************/
struct special_section {
	char *name;
	enum architecture arch;
	int (*group_size)(struct kpatch_elf *kelf, int offset);
	bool (*group_filter)(struct lookup_table *lookup,
			     struct section *relasec, unsigned int offset,
			     unsigned int size);
};

/*************
 * Functions
 * **********/

static bool is_bundleable(struct symbol *sym)
{
	if (sym->type == STT_FUNC &&
	    !strncmp(sym->sec->name, ".text.",6) &&
	    !strcmp(sym->sec->name + 6, sym->name))
		return true;

	if (sym->type == STT_FUNC &&
	    !strncmp(sym->sec->name, ".text.unlikely.",15) &&
	    (!strcmp(sym->sec->name + 15, sym->name) ||
			 (strstr(sym->name, ".cold") &&
			  !strncmp(sym->sec->name + 15, sym->name, strlen(sym->sec->name) - 15))))
		return true;

	if (sym->type == STT_FUNC &&
	    !strncmp(sym->sec->name, ".text.hot.",10) &&
	    !strcmp(sym->sec->name + 10, sym->name))
		return true;

	if (sym->type == STT_OBJECT &&
	   !strncmp(sym->sec->name, ".data.",6) &&
	   !strcmp(sym->sec->name + 6, sym->name))
		return true;

	if (sym->type == STT_OBJECT &&
	    !strncmp(sym->sec->name, ".data.rel.", 10) &&
	    !strcmp(sym->sec->name + 10, sym->name))
		return true;

	if (sym->type == STT_OBJECT &&
	    !strncmp(sym->sec->name, ".data.rel.ro.", 13) &&
	    !strcmp(sym->sec->name + 13, sym->name))
		return true;

	if (sym->type == STT_OBJECT &&
	    !strncmp(sym->sec->name, ".data.rel.ro.local.", 19) &&
	    !strcmp(sym->sec->name + 19, sym->name))
		return 1;

	if (sym->type == STT_OBJECT &&
	    !strncmp(sym->sec->name, ".data.rel.local.", 16) &&
	    !strcmp(sym->sec->name + 16, sym->name))
		return 1;

	if (sym->type == STT_OBJECT &&
	   !strncmp(sym->sec->name, ".rodata.",8) &&
	   !strcmp(sym->sec->name + 8, sym->name))
		return true;

	if (sym->type == STT_OBJECT &&
	   !strncmp(sym->sec->name, ".bss.",5) &&
	   !strcmp(sym->sec->name + 5, sym->name))
		return true;

	return false;
}

/* Symbol st_others value for powerpc */
#define STO_PPC64_LOCAL_BIT     5
#define STO_PPC64_LOCAL_MASK    (7 << STO_PPC64_LOCAL_BIT)
#define PPC64_LOCAL_ENTRY_OFFSET(other)                                 \
	        (((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2)

/*
 * On ppc64le, the function prologue generated by GCC 6+ has the sequence:
 *
 *	.globl my_func
 *	.type my_func, @function
 *	.quad .TOC.-my_func
 * my_func:
 *	.reloc ., R_PPC64_ENTRY ; optional
 *	ld r2,-8(r12)
 *	add r2,r2,r12
 *	.localentry my_func, .-my_func
 *
 * my_func is the global entry point, which, when called, sets up the TOC.
 * .localentry is the local entry point, for calls to the function from within
 * the object file.  The local entry point is 8 bytes after the global entry
 * point.
 */
static bool is_gcc6_localentry_bundled_sym(struct kpatch_elf *kelf,
					  struct symbol *sym)
{
	switch(kelf->arch) {
	case AARCH64:
		return false;
	case PPC64:
		return ((PPC64_LOCAL_ENTRY_OFFSET(sym->sym.st_other) != 0) &&
			sym->sym.st_value == 8);
	case X86_64:
		return false;
	case S390:
		return false;
	default:
		ERROR("unsupported arch");
	}

	return false;
}

/*
 * On ppc64le, when a function references data, it does so indirectly, via the
 * .toc section.  So there are *two* levels of relas:
 *
 *   1) the original function rela, referring to the .toc section; and
 *
 *   2) the .toc section rela, referring to the data needed by the function.
 *
 * For example:
 *
 *   Relocation section '.rela.text.netlink_release' at offset 0xcadf0 contains 44 entries:
 *   ...
 *   0000000000000398  0000007300000032 R_PPC64_TOC16_HA    0000000000000000 .toc + 138
 *   00000000000003a0  0000007300000040 R_PPC64_TOC16_LO_DS 0000000000000000 .toc + 138
 *
 *   Relocation section '.rela.toc' at offset 0xcc6b0 contains 46 entries:
 *   ...
 *   0000000000000138  0000002a00000026 R_PPC64_ADDR64      0000000000000000 .text.deferred_put_nlk_sk + 8
 *
 * The below function takes the "first level" rela as input, and, if it refers
 * to .toc, returns the "second level" rela, which is the one that refers to
 * the actual data symbol.
 *
 * In some rare cases, a .toc entry has constant data, and thus has no
 * corresponding rela.  In that case, NULL is returned.
 */
static struct rela *toc_rela(const struct rela *rela)
{
	if (rela->type != R_PPC64_TOC16_HA &&
	    rela->type != R_PPC64_TOC16_LO_DS)
		return (struct rela *)rela;

	/* Only constants in toc */
	if (!rela->sym->sec->rela)
		return NULL;

	/* Will return NULL for .toc constant entries */
	return find_rela_by_offset(rela->sym->sec->rela,
				   (unsigned int)rela->addend);
}

/*
 * Mapping symbols are used to mark and label the transitions between code and
 * data in elf files. They begin with a "$" dollar symbol. Don't correlate them
 * as they often all have the same name either "$x" to mark the start of code
 * or "$d" to mark the start of data.
 */
static bool kpatch_is_mapping_symbol(struct kpatch_elf *kelf, struct symbol *sym)
{
	switch (kelf->arch) {
	case AARCH64:
		if (sym->name && sym->name[0] == '$'
			&& sym->type == STT_NOTYPE \
			&& sym->bind == STB_LOCAL)
			return true;
	case X86_64:
	case PPC64:
	case S390:
		return false;
	default:
		ERROR("unsupported arch");
	}

	return false;
}

static unsigned int function_padding_size(struct kpatch_elf *kelf, struct symbol *sym)
{
	unsigned int size = 0;

	switch (kelf->arch) {
	case AARCH64:
	{
		uint8_t *insn = sym->sec->data->d_buf;
		unsigned int i;
		void *insn_end = sym->sec->data->d_buf + sym->sym.st_value;

		/*
		 * If the arm64 kernel is compiled with CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
		 * then there are two NOPs before the function and a `BTI C` + 2 NOPs at the
		 * start of the function. Verify the presence of the two NOPs before the
		 * function entry.
		 */
		for (i = 0; (void *)insn < insn_end; i++, insn += 4)
			if (insn[0] != 0x1f || insn[1] != 0x20 ||
			    insn[2] != 0x03 || insn[3] != 0xd5)
				break;

		if (i == 2)
			size = 8;
		else if (i != 0)
			log_error("function %s within section %s has invalid padding\n", sym->name, sym->sec->name);

		break;
	}
	case X86_64: 
	{
		uint8_t *insn = sym->sec->data->d_buf;
		uint8_t *insn_end = sym->sec->data->d_buf + sym->sym.st_value;
	
	       /*
  	        * If the x86_64 kernel is compiled with CONFIG_CALL_PADDING
  	        * then there are CONFIG_FUNCTION_PADDING_BYTES NOPs before the function.Verify the presence of the CONFIG_FUNCTION_PADDING_BYTES 
  	        * NOPs before the function entry.
  	        * Possible padding patterns before X86 functions:
  	        * 1. NOP instructions (0x90) for alignment.
  	        * 2. Multi-byte NOPs that compilers may insert (e.g., 0x66 0x90).
  	        * 3. Jump tables or special instructions may exist before functions (adjust according to actual scenarios).
                */
		while (insn < insn_end) {

			if (*insn == 0x90) {
				size++;
				insn++;
			}
			else if (insn + 1 < insn_end && 
					  insn[0] == 0x66 && insn[1] == 0x90) {
				size += 2;
				insn += 2;
			}
			else {
				break;
			}
		}
	
		if (size > 16) {
			 log_error("function %s in section %s has excessive padding (%u bytes)\n",
					  sym->name, sym->sec->name, size);
			 size = 0;
		}
		break;
	}
	default:
		break;
	}

	return size;
}

/*
 * When compiling with -ffunction-sections and -fdata-sections, almost every
 * symbol gets its own dedicated section.  We call such symbols "bundled"
 * symbols.  They're indicated by "sym->sec->sym == sym".
 */
static void kpatch_bundle_symbols(struct kpatch_elf *kelf)
{
	struct symbol *sym;
	uint64_t expected_offset;

	list_for_each_entry(sym, &kelf->symbols, list) {
		if (is_bundleable(sym)) {
			if (sym->pfx)
				expected_offset = sym->pfx->sym.st_size;
			else if (is_gcc6_localentry_bundled_sym(kelf, sym))
				expected_offset = 8;
			else if (sym->type == STT_FUNC)
				expected_offset = function_padding_size(kelf, sym);
			else
				expected_offset = 0;

			if (sym->sym.st_value != expected_offset) {
				ERROR("symbol %s at offset %lu within section %s, expected %lu",
				      sym->name, sym->sym.st_value,
				      sym->sec->name, expected_offset);
			}

			sym->sec->sym = sym;
		}
	}
}

static struct symbol *kpatch_lookup_parent(struct kpatch_elf *kelf,
					   const char *symname,
					   const char *child_suffix)
{
	struct symbol *parent;
	char *pname;

	pname = strndup(symname, child_suffix - symname);
	if (!pname)
		ERROR("strndup");

	parent = find_symbol_by_name(&kelf->symbols, pname);
	free(pname);

	return parent;
}

/*
 * During optimization gcc may move unlikely execution branches into *.cold
 * subfunctions. Some functions can also be split into multiple *.part
 * functions.
 * kpatch_detect_child_functions detects such subfunctions and
 * crossreferences them with their parent functions through parent/child
 * pointers.
 */
static void kpatch_detect_child_functions(struct kpatch_elf *kelf)
{
	struct symbol *sym;

	list_for_each_entry(sym, &kelf->symbols, list) {
		char *childstr;

		if (sym->type != STT_FUNC)
			continue;

		childstr = strstr(sym->name, ".cold");
		if (childstr) {
			sym->parent = kpatch_lookup_parent(kelf, sym->name,
							   childstr);
			if (!sym->parent)
				ERROR("failed to find parent function for %s",
				      sym->name);
		} else {
			childstr = strstr(sym->name, ".part.");
			if (!childstr)
				continue;
			sym->parent = kpatch_lookup_parent(kelf, sym->name,
							   childstr);
		}

		if (sym->parent)
			list_add_tail(&sym->subfunction_node, &sym->parent->children);
	}
}

static bool is_dynamic_debug_symbol(struct symbol *sym)
{
	if (sym->type == STT_OBJECT && !strcmp(sym->sec->name, "__verbose"))
		return true;
	if (sym->type == STT_OBJECT && !strcmp(sym->sec->name, "__dyndbg"))
		return true;
	if (sym->type == STT_SECTION && !strcmp(sym->name, "__verbose"))
		return true;
	if (sym->type == STT_SECTION && !strcmp(sym->name, "__dyndbg"))
		return true;
	return false;
}

static bool is_string_literal_section(struct section *sec)
{
	return !strncmp(sec->name, ".rodata.", 8) && strstr(sec->name, ".str");
}

/*
 * This function detects whether the given symbol is a "special" static local
 * variable (for lack of a better term).
 *
 * Special static local variables should never be correlated and should always
 * be included if they are referenced by an included function.
 */
static bool is_special_static(struct symbol *sym)
{
	static char *var_names[] = {
		"__key",
		"__warned",
		"__already_done.",
		"__func__",
		"__FUNCTION__",
		"_rs",
		"CSWTCH",
		"_entry",
		NULL,
	};
	char **var_name;

	if (!sym)
		return false;

	/* pr_debug() uses static local variables in the __verbose or __dyndbg section */
	if (is_dynamic_debug_symbol(sym))
		return true;

	if (sym->type == STT_SECTION) {
		/* make sure section is bundled */
		if (!sym->sec->sym)
			return false;

		/* use bundled object/function symbol for matching */
		sym = sym->sec->sym;
	}

	if (sym->type != STT_OBJECT || sym->bind != STB_LOCAL)
		return false;

	if  (!strcmp(sym->sec->name, ".data.once") ||
	     !strcmp(sym->sec->name, ".data..once"))
		return true;

	for (var_name = var_names; *var_name; var_name++) {
		size_t var_name_len = strlen(*var_name);
		char buf[256];

		snprintf(buf, 256, ".%s.", *var_name);

		/* First look for gcc-style statics: '<var_name>.' */
		if (!strncmp(sym->name, buf + 1, var_name_len + 1))
			return true;

		buf[var_name_len + 1] = '\0';
		/* Next clang-style statics: '<function_name>.<var_name>' */
		if (strstr(sym->name, buf))
			return true;
	}

	return false;
}

static bool has_digit_tail(char *tail)
{
	if (*tail != '.')
		return false;

	while (isdigit(*++tail))
		;

	if (!*tail)
		return true;

	return false;
}

/*
 * Hack for __UNIQUE_ID().  The following should match:
 *
 *   __UNIQUE_ID_ddebug1131.186
 *   __UNIQUE_ID_ddebug1132.187
 */
static int __kpatch_unique_id_strcmp(char *s1, char *s2)
{
	/* match '__UNIQUE_ID_ddebug' */
	while (*s1 == *s2) {
		if (!*s1)
			return 0;
		s1++;
		s2++;
	}

	/* skip digits before '.' or EOL */
	while (isdigit(*s1))
		s1++;
	while (isdigit(*s2))
		s2++;

	if ((!*s1 || has_digit_tail(s1)) &&
	    (!*s2 || has_digit_tail(s2)))
		return 0;

	return 1;
}

/*
 * This is like strcmp, but for gcc-mangled symbols.  It skips the comparison
 * of any substring which consists of '.' followed by any number of digits.
 */
static int kpatch_mangled_strcmp(char *s1, char *s2)
{
	/*
	 * ELF string sections aren't mangled, though they look that way.  Just
	 * compare them normally.
	 */
	if (strstr(s1, ".str1."))
		return strcmp(s1, s2);

	if (!strncmp(s1, "__UNIQUE_ID_", 12))
		return __kpatch_unique_id_strcmp(s1, s2);

	/*
	 * With LLVM, dynamic debug unique id looks like
	 *   <func_name>.__UNIQUE_ID_ddebug<number>.
	 *
	 * Use __kpatch_unique_id_strcmp to compare them.
	 */
	if (strstr(s1, "__UNIQUE_ID_ddebug"))
		return __kpatch_unique_id_strcmp(s1, s2);

	while (*s1 == *s2) {
		if (!*s1)
			return 0;
		if (*s1 == '.' && isdigit(s1[1])) {
			if (!isdigit(s2[1]))
				return 1;
			while (isdigit(*++s1))
				;
			while (isdigit(*++s2))
				;
		} else {
			s1++;
			s2++;
		}
	}

	if ((!*s1 && has_digit_tail(s2)) ||
	    (!*s2 && has_digit_tail(s1)))
		return 0;

	return 1;
}

static bool rela_equal(struct rela *rela1, struct rela *rela2)
{
	struct rela *rela_toc1, *rela_toc2;
	unsigned long toc_data1 = 0, toc_data2 = 0; /* = 0 to prevent gcc warning */

	if (rela1->type != rela2->type ||
	    rela1->offset != rela2->offset)
		return false;

	/*
	 * On x86, .altinstr_aux is used to store temporary code which allows
	 * static_cpu_has() to work before apply_alternatives() has run.  This
	 * code is completely inert for modules, because apply_alternatives()
	 * runs during module init, before the module is fully formed.  Any
	 * changed references to it (i.e. changed addend) can be ignored.  As
	 * long as they're both references to .altinstr_aux, they can be
	 * considered equal, even if the addends differ.
	 */
	if (!strcmp(rela1->sym->name, ".altinstr_aux") &&
	    !strcmp(rela2->sym->name, ".altinstr_aux"))
		return true;

	/*
	 * With -mcmodel=large on ppc64le, GCC might generate entries in the .toc
	 * section for relocation symbol references.   The .toc offsets may change
	 * between the original and patched .o, so comparing ".toc + offset" isn't
	 * right.  Compare the .toc-based symbols by reading the corresponding relas
	 * from the .toc section.
	 */
	rela_toc1 = toc_rela(rela1);
	if (!rela_toc1) {
		/*
		 * .toc section entries are mostly place holder for relocation entries, specified
		 * in .rela.toc section. Sometimes, .toc section may have constants as entries.
		 * These constants are not reference to any symbols, but plain instructions mostly
		 * due to some arithmetics in the functions referring them.
		 *
		 * They are referred by the functions like normal .toc entries, these entries can
		 * not be resolved to any symbols.
		 *
		 * Disassembly of section .toc:
		 *
		 * 0000000000000000 <.toc>:
		 *         ...
		 *                        148: R_PPC64_ADDR64     .data.capacity_margin
		 *  150:   0b d7 a3 70     andi.   r3,r5,55051
		 *  154:   3d 0a d7 a3     lhz     r30,2621(r23)
		 *                        158: R_PPC64_ADDR64     sched_max_numa_distance
		 *
		 * Relocation section '.rela.toc' at offset 0xadac0 contains 160 entries:
		 *     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
		 *     ...
		 * 0000000000000148  0000009100000026 R_PPC64_ADDR64         0000000000000000 .data.capacity_margin + 0
		 * 0000000000000158  000001a500000026 R_PPC64_ADDR64         0000000000000000 sched_max_numa_distance + 0
		 *
		 * Relocation section '.rela.text.select_task_rq_fair' at offset 0x90e98 contains 37 entries:
		 *     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
		 *     ...
		 * 00000000000004a0  0000008800000032 R_PPC64_TOC16_HA       0000000000000000 .toc + 148
		 * 00000000000004ac  0000008800000040 R_PPC64_TOC16_LO_DS    0000000000000000 .toc + 148
		 * 0000000000000514  0000008800000032 R_PPC64_TOC16_HA       0000000000000000 .toc + 150
		 * 000000000000051c  0000008800000040 R_PPC64_TOC16_LO_DS    0000000000000000 .toc + 150
		 */
		memcpy(&toc_data1, rela1->sym->sec->data->d_buf + rela1->addend, sizeof(toc_data1));
		if (!toc_data1)
			ERROR(".toc entry not found %s + %lx", rela1->sym->name, rela1->addend);
	}

	rela_toc2 = toc_rela(rela2);
	if (!rela_toc2) {
		memcpy(&toc_data2, rela2->sym->sec->data->d_buf + rela2->addend, sizeof(toc_data2));
		if (!toc_data2)
			ERROR(".toc entry not found %s + %lx", rela2->sym->name, rela2->addend);
	}

	if (!rela_toc1 && !rela_toc2)
		return toc_data1 == toc_data2;

	if (!rela_toc1 || !rela_toc2)
		return false;

	if (rela_toc1->string)
		return rela_toc2->string && !strcmp(rela_toc1->string, rela_toc2->string);

	if (rela_toc1->addend != rela_toc2->addend)
		return false;

	return !kpatch_mangled_strcmp(rela_toc1->sym->name, rela_toc2->sym->name);
}

static void kpatch_compare_correlated_rela_section(struct section *relasec)
{
	struct rela *rela1, *rela2 = NULL;

	/*
	 * On ppc64le, don't compare the .rela.toc section.  The .toc and
	 * .rela.toc sections are included as standard elements.
	 */
	if (!strcmp(relasec->name, ".rela.toc")) {
		relasec->status = SAME;
		return;
	}

	rela2 = list_entry(relasec->twin->relas.next, struct rela, list);
	list_for_each_entry(rela1, &relasec->relas, list) {
		if (rela_equal(rela1, rela2)) {
			rela2 = list_entry(rela2->list.next, struct rela, list);
			continue;
		}
		relasec->status = CHANGED;
		return;
	}

	relasec->status = SAME;
}

static void kpatch_compare_correlated_nonrela_section(struct section *sec)
{
	struct section *sec1 = sec, *sec2 = sec->twin;

	if (sec1->sh.sh_type != SHT_NOBITS &&
	    memcmp(sec1->data->d_buf, sec2->data->d_buf, sec1->data->d_size))
		sec->status = CHANGED;
	else
		sec->status = SAME;
}

static void kpatch_compare_correlated_section(struct section *sec)
{
	struct section *sec1 = sec, *sec2 = sec->twin;

	/* Compare section headers (must match or fatal) */
	if (sec1->sh.sh_type != sec2->sh.sh_type ||
	    sec1->sh.sh_flags != sec2->sh.sh_flags ||
	    sec1->sh.sh_entsize != sec2->sh.sh_entsize ||
	    (sec1->sh.sh_addralign != sec2->sh.sh_addralign &&
	     !is_text_section(sec1)))
		DIFF_FATAL("%s section header details differ from %s", sec1->name, sec2->name);

	/*
	 * Short circuit for mcount and patchable_function_entries
	 * sections, we rebuild regardless
	 */
	if (!strcmp(sec->name, ".rela__mcount_loc") ||
	    !strcmp(sec->name, "__mcount_loc") ||
	    !strcmp(sec->name, ".sframe") ||
	    !strcmp(sec->name, ".rela.sframe") ||
	    !strcmp(sec->name, ".rela__patchable_function_entries") ||
	    !strcmp(sec->name, "__patchable_function_entries")) {
		sec->status = SAME;
		goto out;
	}

	if (sec1->sh.sh_size != sec2->sh.sh_size ||
	    sec1->data->d_size != sec2->data->d_size ||
		(sec1->rela && !sec2->rela) ||
		(sec2->rela && !sec1->rela)) {
		sec->status = CHANGED;
		goto out;
	}

	if (is_rela_section(sec))
		kpatch_compare_correlated_rela_section(sec);
	else
		kpatch_compare_correlated_nonrela_section(sec);
out:
	if (sec->status == CHANGED)
		log_debug("section %s has changed\n", sec->name);
}

/*
 * This function is not comprehensive, i.e. it doesn't detect immediate loads
 * to *all* registers.  It only detects those which have been found in the wild
 * to be involved in the load of a __LINE__ immediate.  If we miss some, that's
 * ok, we'll find them later when someone notices a function falsely being
 * reported as changed ;-)
 *
 * Right now we're only checking immediate loads to the registers corresponding
 * to function arguments 2 and 3 for each respective arch's calling convention.
 * (Argument 1 is typically the printf format string).  Eventually we might
 * want to consider just checking *all* registers which could conceivably be
 * used as function arguments.  But in practice, arg2 and arg3 seem to be the
 * main ones, so for now, take a more conservative approach at the risk of
 * failing to detect some of the more obscure __LINE__-only changed functions.
 */
static bool insn_is_load_immediate(struct kpatch_elf *kelf, void *addr)
{
	unsigned char *insn = addr;

	switch(kelf->arch) {

	case X86_64:
		/* arg2: mov $imm, %esi */
		if (insn[0] == 0xbe)
			return true;

		/* arg3: mov $imm, %edx */
		if (insn[0] == 0xba)
			return true;

		break;

	case PPC64:
		/*
		 * ppc64le insns are LE-encoded:
		 *
		 *   0a 00 80 38     li      r4,10
		 *   47 14 a0 38     li      r5,5191
		 */

		/* arg2: li r4, imm */
		if (insn[3] == 0x38 && insn[2] == 0x80)
			return true;

		/* arg3: li r5, imm */
		if (insn[3] == 0x38 && insn[2] == 0xa0)
			return true;

		break;

	case S390:
		/* arg2: lghi %r3, imm */
		if (insn[0] == 0xa7 && insn[1] == 0x39)
			return true;

		/* arg3: lghi %r4, imm */
		if (insn[0] == 0xa7 && insn[1] == 0x49)
			return true;

		break;

	case AARCH64:
		/* Verify mov w2 <line number> */
		if ((insn[0] & 0b11111) == 0x2 && insn[3] == 0x52)
			return true;
		break;

	default:
		ERROR("unsupported arch");
	}

	return false;
}

/*
 * Determine if a section has changed only due to a __LINE__ number change,
 * e.g. a WARN() or might_sleep() macro's embedding of the line number into an
 * instruction operand.
 *
 * Warning: Hackery lies herein.  It's hopefully justified by the fact that
 * this issue is very common.
 *
 * Example WARN():
 *
 *  938:   be 70 00 00 00          mov    $0x70,%esi
 *  93d:   48 c7 c7 00 00 00 00    mov    $0x0,%rdi
 *                         940: R_X86_64_32S       .rodata.tcp_conn_request.str1.8+0x88
 *  944:   c6 05 00 00 00 00 01    movb   $0x1,0x0(%rip)        # 94b <tcp_conn_request+0x94b>
 *                         946: R_X86_64_PC32      .data.unlikely-0x1
 *  94b:   e8 00 00 00 00          callq  950 <tcp_conn_request+0x950>
 *                         94c: R_X86_64_PC32      warn_slowpath_null-0x4
 *
 * Example might_sleep:
 *
 *  50f:   be f7 01 00 00          mov    $0x1f7,%esi
 *  514:   48 c7 c7 00 00 00 00    mov    $0x0,%rdi
 *                         517: R_X86_64_32S       .rodata.do_select.str1.8+0x98
 *  51b:   e8 00 00 00 00          callq  520 <do_select+0x520>
 *                         51c: R_X86_64_PC32      ___might_sleep-0x4
 */
static bool kpatch_line_macro_change_only(struct kpatch_elf *kelf,
					  struct section *sec)
{
	unsigned long offset, insn1_len, insn2_len;
	void *data1, *data2, *insn1, *insn2;
	struct rela *r, *rela;
	bool found, found_any = false;
	bool warn_printk_only = (kelf->arch == AARCH64);

	if (sec->status != CHANGED ||
	    is_rela_section(sec) ||
	    !is_text_section(sec) ||
	    sec->sh.sh_size != sec->twin->sh.sh_size ||
	    !sec->rela ||
	    sec->rela->status != SAME)
		return false;

	data1 = sec->twin->data->d_buf;
	data2 = sec->data->d_buf;
	for (offset = 0; offset < sec->sh.sh_size; offset += insn1_len) {

		insn1 = data1 + offset;
		insn2 = data2 + offset;

		insn1_len = insn_length(kelf, insn1);
		insn2_len = insn_length(kelf, insn2);

		if (!insn1_len || !insn2_len)
			ERROR("can't decode instruction in section %s at offset 0x%lx",
			      sec->name, offset);

		if (insn1_len != insn2_len)
			return false;

		if (!memcmp(insn1, insn2, insn1_len))
			continue;

		/*
		 * Here we found a difference between two instructions of the
		 * same length.  Only ignore the change if:
		 *
		 *   a) the instructions match a known pattern of a '__LINE__'
		 *      macro immediate value which was embedded in the
		 *      instruction; and
		 *
		 *   b) the instructions are followed by certain expected
		 *      relocations.
		 */

		if (!insn_is_load_immediate(kelf, insn1) ||
		    !insn_is_load_immediate(kelf, insn2))
			return false;

		found = false;
		list_for_each_entry(r, &sec->rela->relas, list) {

			if (r->offset < offset + insn1_len)
				continue;

			rela = toc_rela(r);
			if (!rela)
				continue;

			if (rela->string)
				continue;

			if (!strncmp(rela->sym->name, "__warned.", 9) ||
			    !strncmp(rela->sym->name, "__already_done.", 15) ||
			    !strncmp(rela->sym->name, "__func__.", 9))
				continue;

			if (!strcmp(rela->sym->name, "__warn_printk")) {
				found = true;
				break;
			}

			if (warn_printk_only)
				return false;

			if (!strncmp(rela->sym->name, "warn_slowpath_", 14) ||
			    !strcmp(rela->sym->name, "__might_sleep") ||
			    !strcmp(rela->sym->name, "___might_sleep") ||
			    !strcmp(rela->sym->name, "__might_fault") ||
			    !strcmp(rela->sym->name, "printk") ||
			    !strcmp(rela->sym->name, "_printk") ||
			    !strcmp(rela->sym->name, "lockdep_rcu_suspicious") ||
			    !strcmp(rela->sym->name, "__btrfs_abort_transaction") ||
			    !strcmp(rela->sym->name, "__btrfs_handle_fs_error") ||
			    !strcmp(rela->sym->name, "__btrfs_panic")) {
				found = true;
				break;
			}
			return false;
		}
		if (!found)
			return false;

		found_any = true;
	}

	if (!found_any)
		ERROR("no instruction changes detected for changed section %s",
		      sec->name);

	return true;
}

/*
 * Child functions with "*.cold" names don't have _fentry_ calls, but "*.part",
 * often do. In the later case, it is not necessary to include the parent
 * in the output object when the child function has changed.
 */
static bool kpatch_changed_child_needs_parent_profiling(struct symbol *sym)
{
	struct symbol *child;

	list_for_each_entry(child, &sym->children, subfunction_node) {
		if (child->has_func_profiling)
			continue;
		if (child->sec->status == CHANGED ||
		    kpatch_changed_child_needs_parent_profiling(child))
			return true;
	}

	return false;
}

static void kpatch_compare_sections(struct kpatch_elf *kelf)
{
	struct section *sec;
	struct list_head *seclist = &kelf->sections;

	/* compare all sections */
	list_for_each_entry(sec, seclist, list) {
		if (sec->twin)
			kpatch_compare_correlated_section(sec);
		else
			sec->status = NEW;
	}

	/* exclude WARN-only, might_sleep changes */
	list_for_each_entry(sec, seclist, list) {
		if (kpatch_line_macro_change_only(kelf, sec)) {
			log_debug("reverting macro / line number section %s status to SAME\n",
				  sec->name);
			sec->status = SAME;
		}
	}

	/* sync symbol status */
	list_for_each_entry(sec, seclist, list) {
		if (is_rela_section(sec)) {
			if (sec->base->sym && sec->base->sym->status != CHANGED)
				sec->base->sym->status = sec->status;
		} else {
			struct symbol *sym = sec->sym;

			if (sym && sym->status != CHANGED)
				sym->status = sec->status;

			if (sym && sym->status == SAME &&
			    kpatch_changed_child_needs_parent_profiling(sym))
				sym->status = CHANGED;
		}
	}
}

static enum subsection kpatch_subsection_type(struct section *sec)
{
	if (!strncmp(sec->name, ".text.unlikely.", 15))
		return SUBSECTION_UNLIKELY;

	if (!strncmp(sec->name, ".text.hot.", 10))
		return SUBSECTION_HOT;

	return SUBSECTION_NORMAL;
}

static bool kpatch_subsection_changed(struct section *sec1, struct section *sec2)
{
	return kpatch_subsection_type(sec1) != kpatch_subsection_type(sec2);
}

static struct symbol *kpatch_get_correlated_parent(struct symbol *sym)
{
	while (sym->parent && !sym->parent->twin)
		sym = sym->parent;
	return sym->parent;
}

static void kpatch_compare_correlated_symbol(struct symbol *sym)
{
	struct symbol *sym1 = sym, *sym2 = sym->twin;

	if (sym1->sym.st_info != sym2->sym.st_info ||
	    (sym1->sec && !sym2->sec) ||
	    (sym2->sec && !sym1->sec))
		DIFF_FATAL("symbol info mismatch: %s", sym1->name);

	/*
	 * If two symbols are correlated but their sections are not, then the
	 * symbol has changed sections.  This is only allowed if the symbol is
	 * moving out of an ignored section, or moving between normal/hot/unlikely
	 * subsections.
	 */
	if (sym1->sec && sym2->sec && sym1->sec->twin != sym2->sec) {
		if ((sym2->sec->twin && sym2->sec->twin->ignore) ||
		    kpatch_subsection_changed(sym1->sec, sym2->sec))
			sym->status = CHANGED;
		else
			DIFF_FATAL("symbol changed sections: %s", sym1->name);
	}

	if (sym1->type == STT_OBJECT &&
	    sym1->sym.st_size != sym2->sym.st_size)
		DIFF_FATAL("object size mismatch: %s", sym1->name);

	if (sym1->sym.st_shndx == SHN_UNDEF ||
	     sym1->sym.st_shndx == SHN_ABS)
		sym1->status = SAME;

	/*
	 * The status of LOCAL symbols is dependent on the status of their
	 * matching section and is set during section comparison.
	 */
}

static void kpatch_compare_symbols(struct list_head *symlist)
{
	struct symbol *sym;

	list_for_each_entry(sym, symlist, list) {
		if (sym->twin)
			kpatch_compare_correlated_symbol(sym);
		else
			sym->status = NEW;

		log_debug("symbol %s is %s\n", sym->name, status_str(sym->status));
	}
}

#define CORRELATE_ELEMENT(_e1_, _e2_, kindstr)			\
do {								\
	typeof(_e1_) e1 = (_e1_);				\
	typeof(_e2_) e2 = (_e2_);				\
	e1->twin = e2;						\
	e2->twin = e1;						\
	/* set initial status, might change */			\
	e1->status = e2->status = SAME;				\
	if (strcmp(e1->name, e2->name)) {			\
		/* Rename mangled element */			\
		log_debug("renaming %s %s to %s\n",		\
			  kindstr, e2->name, e1->name);		\
		e2->name = strdup(e1->name);			\
		if (!e2->name)				        \
			ERROR("strdup");			\
	}							\
} while (0)

#define UNCORRELATE_ELEMENT(_elem_)				\
do {								\
	typeof(_elem_) elem = (_elem_);				\
	elem->twin->twin = NULL;				\
	elem->twin = NULL;					\
} while (0)

static void __kpatch_correlate_section(struct section *sec_orig,
		struct section *sec_patched)
{
	CORRELATE_ELEMENT(sec_orig, sec_patched, "section");
}

static void kpatch_correlate_symbol(struct symbol *sym_orig,
		struct symbol *sym_patched)
{
	CORRELATE_ELEMENT(sym_orig, sym_patched, "symbol");
	if (sym_orig->lookup_table_file_sym && !sym_patched->lookup_table_file_sym)
		sym_patched->lookup_table_file_sym = sym_orig->lookup_table_file_sym;
}

static void kpatch_correlate_static_local(struct symbol *sym_orig,
		struct symbol *sym_patched)
{
	CORRELATE_ELEMENT(sym_orig, sym_patched, "static local");
}

static void kpatch_correlate_section(struct section *sec_orig,
		struct section *sec_patched)
{
	__kpatch_correlate_section(sec_orig, sec_patched);

	if (is_rela_section(sec_orig)) {
		__kpatch_correlate_section(sec_orig->base, sec_patched->base);
		sec_orig = sec_orig->base;
		sec_patched = sec_patched->base;
	} else if (sec_orig->rela && sec_patched->rela) {
		__kpatch_correlate_section(sec_orig->rela, sec_patched->rela);
	}

	if (sec_orig->secsym)
		kpatch_correlate_symbol(sec_orig->secsym, sec_patched->secsym);
	if (sec_orig->sym)
		kpatch_correlate_symbol(sec_orig->sym, sec_patched->sym);
}

static void kpatch_correlate_sections(struct list_head *seclist_orig,
		struct list_head *seclist_patched)
{
	struct section *sec_orig, *sec_patched;

	list_for_each_entry(sec_orig, seclist_orig, list) {
		if (sec_orig->twin)
			continue;
		list_for_each_entry(sec_patched, seclist_patched, list) {
			if (kpatch_mangled_strcmp(sec_orig->name, sec_patched->name) ||
			    sec_patched->twin)
				continue;

			if (is_ubsan_sec(sec_orig->name))
				continue;

			if (is_special_static(is_rela_section(sec_orig) ?
					      sec_orig->base->secsym :
					      sec_orig->secsym))
				continue;

			/*
			 * Group sections must match exactly to be correlated.
			 * Changed group sections are currently not supported.
			 */
			if (sec_orig->sh.sh_type == SHT_GROUP) {
				if (sec_orig->data->d_size != sec_patched->data->d_size)
					continue;
				if (memcmp(sec_orig->data->d_buf, sec_patched->data->d_buf,
				           sec_orig->data->d_size))
					continue;
			}

			kpatch_correlate_section(sec_orig, sec_patched);
			break;
		}
	}
}

static void kpatch_correlate_symbols(struct kpatch_elf *kelf_orig,
		struct kpatch_elf *kelf_patched)
{
	struct symbol *sym_orig, *sym_patched;

	list_for_each_entry(sym_orig, &kelf_orig->symbols, list) {
		if (sym_orig->twin)
			continue;
		list_for_each_entry(sym_patched, &kelf_patched->symbols, list) {
			if (kpatch_mangled_strcmp(sym_orig->name, sym_patched->name) ||
			    sym_orig->type != sym_patched->type || sym_patched->twin)
				continue;

			if (is_ubsan_sec(sym_orig->name))
				continue;

			if (is_special_static(sym_orig))
				continue;

			/*
			 * The .LCx symbols point to string literals in
			 * '.rodata.<func>.str1.*' sections.  They get included
			 * in kpatch_include_standard_elements().
			 */
			if (sym_orig->type == STT_NOTYPE &&
			    !strncmp(sym_orig->name, ".LC", 3))
				continue;

			if (kpatch_is_mapping_symbol(kelf_orig, sym_orig))
				continue;

			/* group section symbols must have correlated sections */
			if (sym_orig->sec &&
			    sym_orig->sec->sh.sh_type == SHT_GROUP &&
			    sym_orig->sec->twin != sym_patched->sec)
				continue;

			kpatch_correlate_symbol(sym_orig, sym_patched);
			break;
		}
	}
}

static void kpatch_compare_elf_headers(Elf *elf_orig, Elf *elf_patched)
{
	GElf_Ehdr ehdr_orig, ehdr_patched;

	if (!gelf_getehdr(elf_orig, &ehdr_orig))
		ERROR("gelf_getehdr");

	if (!gelf_getehdr(elf_patched, &ehdr_patched))
		ERROR("gelf_getehdr");

	if (memcmp(ehdr_orig.e_ident, ehdr_patched.e_ident, EI_NIDENT) ||
	    ehdr_orig.e_type != ehdr_patched.e_type ||
	    ehdr_orig.e_machine != ehdr_patched.e_machine ||
	    ehdr_orig.e_version != ehdr_patched.e_version ||
	    ehdr_orig.e_entry != ehdr_patched.e_entry ||
	    ehdr_orig.e_phoff != ehdr_patched.e_phoff ||
	    ehdr_orig.e_flags != ehdr_patched.e_flags ||
	    ehdr_orig.e_ehsize != ehdr_patched.e_ehsize ||
	    ehdr_orig.e_phentsize != ehdr_patched.e_phentsize ||
	    ehdr_orig.e_shentsize != ehdr_patched.e_shentsize)
		DIFF_FATAL("ELF headers differ");
}

static void kpatch_check_program_headers(Elf *elf)
{
	size_t ph_nr;

	if (elf_getphdrnum(elf, &ph_nr))
		ERROR("elf_getphdrnum");

	if (ph_nr != 0)
		DIFF_FATAL("ELF contains program header");
}

static void kpatch_mark_grouped_sections(struct kpatch_elf *kelf)
{
	struct section *groupsec, *sec;
	unsigned int *data, *end;

	list_for_each_entry(groupsec, &kelf->sections, list) {
		if (groupsec->sh.sh_type != SHT_GROUP)
			continue;
		data = groupsec->data->d_buf;
		end = groupsec->data->d_buf + groupsec->data->d_size;
		data++; /* skip first flag word (e.g. GRP_COMDAT) */
		while (data < end) {
			sec = find_section_by_index(&kelf->sections, *data);
			if (!sec)
				ERROR("group section not found");
			sec->grouped = 1;
			log_debug("marking section %s (%d) as grouped\n",
			          sec->name, sec->index);
			data++;
		}
	}
}

static char *kpatch_section_function_name(struct section *sec)
{
	if (is_rela_section(sec))
		sec = sec->base;
	return sec->sym ? sec->sym->name : sec->name;
}

static struct symbol *kpatch_find_uncorrelated_rela(struct section *relasec,
						    struct symbol *sym)
{
	struct rela *rela, *rela_toc;

	/* find the patched object's corresponding variable */
	list_for_each_entry(rela, &relasec->relas, list) {
		struct symbol *patched_sym;

		rela_toc = toc_rela(rela);
		if (!rela_toc)
			continue; /* skip toc constants */

		patched_sym = rela_toc->sym;

		if (patched_sym->twin)
			continue;

		if (sym->type != patched_sym->type ||
		    (sym->type == STT_OBJECT &&
		     sym->sym.st_size != patched_sym->sym.st_size))
			continue;

		if (kpatch_mangled_strcmp(patched_sym->name, sym->name))
			continue;

		return patched_sym;
	}

	return NULL;
}

static struct symbol *kpatch_find_static_twin_in_children(struct symbol *parent,
							  struct symbol *sym)
{
	struct symbol *child;

	list_for_each_entry(child, &parent->children, subfunction_node) {
		struct symbol *res;

		/* Only look in children whose rela section differ from the parent's */
		if (child->sec->rela == parent->sec->rela || !child->sec->rela)
			continue;

		res = kpatch_find_uncorrelated_rela(child->sec->rela, sym);
		/* Need to go deeper */
		if (!res)
			res = kpatch_find_static_twin_in_children(child, sym);
		if (res != NULL)
			return res;
	}

	return NULL;
}

/*
 * Given a static local variable symbol and a rela section which references it
 * in the base object, find a corresponding usage of a similarly named symbol
 * in the patched object.
 */
static struct symbol *kpatch_find_static_twin(struct section *relasec,
					      struct symbol *sym)
{
	struct symbol *res;

	if (!relasec->twin && relasec->base->sym) {
		struct symbol *parent = NULL;

		/*
		 * The static twin might have been in a .part. symbol in the
		 * original object that got removed in the patched object.
		 */
		parent = kpatch_get_correlated_parent(relasec->base->sym);
		if (parent)
			relasec = parent->sec->rela;

	}

	if (!relasec->twin)
		return NULL;

	res = kpatch_find_uncorrelated_rela(relasec->twin, sym);
	if (res != NULL)
		return res;

	/* Look if reference might have moved to child functions' sections */
	if (relasec->twin->base->sym)
		return kpatch_find_static_twin_in_children(relasec->twin->base->sym,
							   sym);

	return NULL;
}

static bool kpatch_is_normal_static_local(struct symbol *sym)
{
	if (sym->type != STT_OBJECT || sym->bind != STB_LOCAL)
		return false;

	if (!strncmp(sym->name, ".L", 2))
		return false;

	if (!strchr(sym->name, '.'))
		return false;

	if (is_special_static(sym))
		return false;

	return true;
}

static struct rela *kpatch_find_static_twin_ref(struct section *relasec,
						struct symbol *sym)
{
	struct rela *rela;

	list_for_each_entry(rela, &relasec->relas, list) {
		if (rela->sym == sym->twin)
			return rela;
	}

	/* Reference to static variable might have moved to child function section */
	if (relasec->base->sym) {
		struct symbol *parent = relasec->base->sym;
		struct symbol *child;

		list_for_each_entry(child, &parent->children, subfunction_node) {
			/* Only look in children whose rela section differ from the parent's */
			if (child->sec->rela == parent->sec->rela ||
			    !child->sec->rela)
				continue;

			rela = kpatch_find_static_twin_ref(child->sec->rela, sym);
			if (rela)
				return rela;
		}
	}

	return NULL;
}

/*
 * gcc renames static local variables by appending a period and a number.  For
 * example, __foo could be renamed to __foo.31452.  Unfortunately this number
 * can arbitrarily change.  Correlate them by comparing which functions
 * reference them, and rename the patched symbols to match the base symbol
 * names.
 *
 * Some surprising facts about static local variable symbols:
 *
 * - It's possible for multiple functions to use the same
 *   static local variable if the variable is defined in an
 *   inlined function.
 *
 * - It's also possible for multiple static local variables
 *   with the same name to be used in the same function if they
 *   have different scopes.  (We have to assume that in such
 *   cases, the order in which they're referenced remains the
 *   same between the orig and patched objects, as there's no
 *   other way to distinguish them.)
 *
 * - Static locals are usually referenced by functions, but
 *   they can occasionally be referenced by data sections as
 *   well.
 */
static void kpatch_correlate_static_local_variables(struct kpatch_elf *orig,
						    struct kpatch_elf *patched)
{
	struct symbol *sym, *patched_sym;
	struct section *relasec;
	struct rela *rela;
	int bundled, patched_bundled;

	/*
	 * First undo the correlations for all static locals.  Two static
	 * locals can have the same numbered suffix in the orig and patched
	 * objects by coincidence.
	 */
	list_for_each_entry(sym, &orig->symbols, list) {

		if (!kpatch_is_normal_static_local(sym))
			continue;

		if (sym->twin)
			UNCORRELATE_ELEMENT(sym);

		bundled = sym == sym->sec->sym;
		if (bundled && sym->sec->twin) {
			UNCORRELATE_ELEMENT(sym->sec);

			if (sym->sec->secsym)
				UNCORRELATE_ELEMENT(sym->sec->secsym);

			if (sym->sec->rela)
				UNCORRELATE_ELEMENT(sym->sec->rela);
		}
	}

	/*
	 * Do the correlations: for each section reference to a static local,
	 * look for a corresponding reference in the section's twin.
	 */
	list_for_each_entry(relasec, &orig->sections, list) {

		if (!is_rela_section(relasec) ||
		    is_debug_section(relasec) ||
		    !strcmp(relasec->name, ".rela.toc"))
			continue;

		list_for_each_entry(rela, &relasec->relas, list) {

			if (!toc_rela(rela))
				continue; /* skip toc constants */
			sym = toc_rela(rela)->sym;

			if (!kpatch_is_normal_static_local(sym))
				continue;

			if (sym->twin)
				continue;

			bundled = sym == sym->sec->sym;
			if (bundled && sym->sec == relasec->base) {
				/*
				 * A rare case where a static local data
				 * structure references itself.  There's no
				 * reliable way to correlate this.  Hopefully
				 * there's another reference to the symbol
				 * somewhere that can be used.
				 */
				log_debug("can't correlate static local %s's reference to itself\n",
					  sym->name);
				continue;
			}

			patched_sym = kpatch_find_static_twin(relasec, sym);
			if (!patched_sym)
				DIFF_FATAL("reference to static local variable %s in %s was removed",
					   sym->name,
					   kpatch_section_function_name(relasec));

			patched_bundled = patched_sym == patched_sym->sec->sym;
			if (bundled != patched_bundled)
				ERROR("bundle mismatch for symbol %s", sym->name);
			if (!bundled && sym->sec->twin != patched_sym->sec)
				ERROR("sections %s and %s aren't correlated for symbol %s",
				      sym->sec->name, patched_sym->sec->name, sym->name);

			kpatch_correlate_static_local(sym, patched_sym);

			if (bundled)
				kpatch_correlate_section(sym->sec, patched_sym->sec);
		}
	}

	/*
	 * Make sure that:
	 *
	 * 1. all the orig object's referenced static locals have been
	 *    correlated; and
	 *
	 * 2. each reference to a static local in the orig object has a
	 *    corresponding reference in the patched object (because a static
	 *    local can be referenced by more than one section).
	 */
	list_for_each_entry(relasec, &orig->sections, list) {

		if (!is_rela_section(relasec) ||
		    is_debug_section(relasec))
			continue;

		list_for_each_entry(rela, &relasec->relas, list) {
			struct section *target_sec = relasec;

			sym = rela->sym;
			if (!kpatch_is_normal_static_local(sym))
				continue;

			if (!relasec->twin && relasec->base->sym) {
				struct symbol *parent = NULL;

				parent = kpatch_get_correlated_parent(relasec->base->sym);
				if (parent)
					target_sec = parent->sec->rela;
			}

			if (!sym->twin || !target_sec->twin)
				DIFF_FATAL("reference to static local variable %s in %s was removed",
					   sym->name,
					   kpatch_section_function_name(target_sec));

			if (!kpatch_find_static_twin_ref(target_sec->twin, sym))
				DIFF_FATAL("static local %s has been correlated with %s, but patched %s is missing a reference to it",
					   sym->name, sym->twin->name,
					   kpatch_section_function_name(target_sec->twin));
		}
	}

	/*
	 * Now go through the patched object and look for any uncorrelated
	 * static locals to see if we need to print any warnings about new
	 * variables.
	 */
	list_for_each_entry(relasec, &patched->sections, list) {

		if (!is_rela_section(relasec) ||
		    is_debug_section(relasec))
			continue;

		list_for_each_entry(rela, &relasec->relas, list) {

			sym = rela->sym;
			if (!kpatch_is_normal_static_local(sym))
				continue;

			if (sym->twin)
				continue;

			log_normal("WARNING: unable to correlate static local variable %s used by %s, assuming variable is new\n",
				   sym->name,
				   kpatch_section_function_name(relasec));
			return;
		}
	}
}

static void kpatch_correlate_elfs(struct kpatch_elf *kelf_orig,
		struct kpatch_elf *kelf_patched)
{
	kpatch_correlate_sections(&kelf_orig->sections, &kelf_patched->sections);
	kpatch_correlate_symbols(kelf_orig, kelf_patched);
}

static void kpatch_compare_correlated_elements(struct kpatch_elf *kelf)
{
	/* lists are already correlated at this point */
	kpatch_compare_sections(kelf);
	kpatch_compare_symbols(&kelf->symbols);
}

static bool is_callback_section(struct section *sec) {

	static char *callback_sections[] = {
		".kpatch.callbacks.pre_patch",
		".kpatch.callbacks.post_patch",
		".kpatch.callbacks.pre_unpatch",
		".kpatch.callbacks.post_unpatch",
		".rela.kpatch.callbacks.pre_patch",
		".rela.kpatch.callbacks.post_patch",
		".rela.kpatch.callbacks.pre_unpatch",
		".rela.kpatch.callbacks.post_unpatch",
		NULL,
	};
	char **callback_sec;

	for (callback_sec = callback_sections; *callback_sec; callback_sec++)
		if (!strcmp(sec->name, *callback_sec))
			return true;

	return false;
}

/*
 * Mangle the relas a little.  The compiler will sometimes use section symbols
 * to reference local objects and functions rather than the object or function
 * symbols themselves.  We substitute the object/function symbols for the
 * section symbol in this case so that the relas can be properly correlated and
 * so that the existing object/function in vmlinux can be linked to.
 */
static void kpatch_replace_sections_syms(struct kpatch_elf *kelf)
{
	struct section *relasec;
	struct rela *rela;
	struct symbol *sym;
	long target_off;
	bool found = false;

	log_debug("\n");

	list_for_each_entry(relasec, &kelf->sections, list) {
		if (!is_rela_section(relasec) || is_debug_section(relasec))
			continue;

		/*
		 * We regenerate __patchable_function_entries from scratch so
		 * don't bother replacing section symbols in its relasec.
		 */
		if (is_patchable_function_entries_section(relasec))
			continue;

		list_for_each_entry(rela, &relasec->relas, list) {

			if (rela->sym->type != STT_SECTION || !rela->sym->sec)
				continue;

			/*
			 * UBSAN data will be taken wholesale, no need to
			 * replace section symbols.
			 */
			if (is_ubsan_sec(rela->sym->name))
				continue;

			/*
			 * These sections don't have symbols associated with
			 * them:
			 */
			if (!strcmp(rela->sym->name, ".toc") ||
			    !strcmp(rela->sym->name, ".fixup") ||
			    !strcmp(rela->sym->name, ".altinstr_replacement") ||
			    !strcmp(rela->sym->name, ".altinstr_aux") ||
			    !strcmp(rela->sym->name, ".text..refcount") ||
			    !strncmp(rela->sym->name, "__ftr_alt_", 10))
				continue;

			/*
			 * Replace references to bundled sections with their
			 * symbols.
			 */
			if (rela->sym->sec->sym) {
				rela->sym = rela->sym->sec->sym;

				/*
				 * On ppc64le with GCC6+, even with
				 * -ffunction-sections, the function symbol
				 *  starts 8 bytes past the beginning of the
				 *  section, because the .TOC pointer is at the
				 *  beginning, right before the code.  So even
				 *  though the symbol is bundled, we can't
				 *  assume it's at offset 0 in the section.
				 */
				rela->addend -= rela->sym->sym.st_value;

				continue;
			}

			target_off = rela_target_offset(kelf, relasec, rela);

			/*
			 * Attempt to replace references to unbundled sections
			 * with their symbols.
			 */
			list_for_each_entry(sym, &kelf->symbols, list) {
				long start, end;

				if (sym->type == STT_SECTION ||
				    sym->sec != rela->sym->sec)
					continue;

				start = sym->sym.st_value;
				end = sym->sym.st_value + sym->sym.st_size;

				if (is_text_section(relasec->base) &&
				    !is_text_section(sym->sec) &&
				    (rela->type == R_X86_64_32S ||
				     rela->type == R_AARCH64_ABS64) &&
				    rela->addend == (long)sym->sec->sh.sh_size &&
				    end == (long)sym->sec->sh.sh_size) {

					/*
					 * A special case where gcc needs a
					 * pointer to the address at the end of
					 * a data section.
					 *
					 * This is usually used with a compare
					 * instruction to determine when to end
					 * a loop.  The code doesn't actually
					 * dereference the pointer so this is
					 * "normal" and we just replace the
					 * section reference with a reference
					 * to the last symbol in the section.
					 *
					 * Note that this only catches the
					 * issue when it happens at the end of
					 * a section.  It can also happen in
					 * the middle of a section.  In that
					 * case, the wrong symbol will be
					 * associated with the reference.  But
					 * that's ok because:
					 *
					 * 1) This situation only occurs when
					 *    gcc is trying to get the address
					 *    of the symbol, not the contents
					 *    of its data; and
					 *
					 * 2) Because kpatch doesn't allow data
					 *    sections to change,
					 *    &(var1+sizeof(var1)) will always
					 *    be the same as &var2.
					 */
				} else if (target_off == start && target_off == end) {

					if(kpatch_is_mapping_symbol(kelf, sym))
						continue;

					/*
					 * Allow replacement for references to
					 * empty symbols.
					 */

				} else if (target_off < start || target_off >= end)
					continue;

				log_debug("%s: replacing %s+%ld reference with %s+%ld\n",
					  relasec->name,
					  rela->sym->name, rela->addend,
					  sym->name, rela->addend - start);
				found = true;

				rela->sym = sym;
				rela->addend -= start;
				break;
			}

			if (!found && !is_string_literal_section(rela->sym->sec) &&
			    strncmp(rela->sym->name, ".rodata", 7)) {
				ERROR("%s+0x%x: can't find replacement symbol for %s+%ld reference",
				      relasec->base->name, rela->offset, rela->sym->name, rela->addend);
			}
		}
	}
	log_debug("\n");
}

static void kpatch_check_func_profiling_calls(struct kpatch_elf *kelf)
{
	struct symbol *sym;
	int errs = 0;

	list_for_each_entry(sym, &kelf->symbols, list) {
		if (sym->type != STT_FUNC || sym->status != CHANGED ||
		    (sym->parent && sym->parent->status == CHANGED))
			continue;
		if (!sym->twin->has_func_profiling) {
			log_error("function %s doesn't have patchable function entry, unable to patch\n",
				   sym->name);
			errs++;
		}
	}

	if (errs)
		DIFF_FATAL("%d function(s) can not be patched", errs);
}

static void kpatch_verify_patchability(struct kpatch_elf *kelf)
{
	struct section *sec;
	int errs = 0;

	list_for_each_entry(sec, &kelf->sections, list) {
		if (sec->status == CHANGED && !sec->include) {
			log_error("changed section %s not selected for inclusion\n",
				  sec->name);
			errs++;
		}

		if (sec->status != SAME && sec->grouped) {
			log_error("changed section %s is part of a section group\n",
				  sec->name);
			errs++;
		}

		if (sec->sh.sh_type == SHT_GROUP && sec->status == NEW) {
			log_error("new/changed group sections are not supported\n");
			errs++;
		}

		/*
		 * ensure we aren't including .data.* or .bss.*
		 * (.data.unlikely, .data..unlikely, .data.once, and .data..once is ok
		 * b/c it only has __warned vars)
		 */
		if (sec->include && sec->status != NEW &&
		    (!strncmp(sec->name, ".data", 5) || !strncmp(sec->name, ".bss", 4)) &&
		    (strcmp(sec->name, ".data.unlikely") && strcmp(sec->name, ".data..unlikely") &&
		     strcmp(sec->name, ".data.once") && strcmp(sec->name, ".data..once"))) {
			log_error("data section %s selected for inclusion\n",
				   sec->name);
			errs++;
		}
	}

	if (errs)
		DIFF_FATAL("%d unsupported section change(s)", errs);
}

static void kpatch_include_symbol(struct symbol *sym);

static void kpatch_include_section(struct section *sec)
{
	struct rela *rela;

	/* Include the section and its section symbol */
	if (sec->include)
		return;
	sec->include = 1;
	if (sec->secsym)
		sec->secsym->include = 1;

	/*
	 * Include the section's rela section and then recursively include the
	 * symbols needed by its relas.
	 */
	if (!sec->rela)
		return;
	sec->rela->include = 1;
	list_for_each_entry(rela, &sec->rela->relas, list)
		kpatch_include_symbol(rela->sym);
}

static void kpatch_include_symbol(struct symbol *sym)
{
	/*
	 * This function is called recursively from kpatch_include_section().
	 * Make sure we don't get into an endless loop.
	 */
	if (sym->include)
		return;

	/*
	 * The symbol gets included even if its section isn't needed, as it
	 * might be needed: either permanently for a rela, or temporarily for
	 * the later creation of a klp relocation.
	 */
	sym->include = 1;

	/*
	 * For a function/object symbol, if it has a section, we only need to
	 * include the section if it has changed.  Otherwise the symbol will be
	 * used by relas/klp_relocs to link to the real symbol externally.
	 *
	 * For section symbols, we always include the section because
	 * references to them can't otherwise be resolved externally.
	 */
	if (sym->sec && (sym->type == STT_SECTION || sym->status != SAME))
		kpatch_include_section(sym->sec);
}

/*
 * Module namespaces are saved as key value pairs in the .modinfo
 * section.  For modules, they are in the form "import_ns=<value" and
 * for built-ins, "<object>.import_ns=<value>".  Extract just these
 * namespace strings from the patched .modinfo to put back into the
 * output ELF.
 */
static void kpatch_get_modinfo_namespaces(struct kpatch_elf *kelf_patched,
					  struct section *sec)
{
	char *data = NULL;
	int data_len = 0;
	const char *p, *end;

	if (!sec->data || !sec->data->d_buf)
		return;

       /*
        * Copy all the "import_ns=<value>" strings into
        * a temporary data array (ignore other key-values)
        */
	p = sec->data->d_buf;
	end = p + sec->sh.sh_size;

	while (p < end) {
		const char *match;
		char *new_data;
		size_t match_len;

		match = memmem(p, end - p, "import_ns=", 10);
		if (!match)
			break;
		match_len = strlen(match) + 1;

		new_data = realloc(data, data_len + match_len);
		if (!new_data)
			ERROR("realloc");
		data = new_data;

		memcpy(data + data_len, match, match_len);
		data_len += (int) match_len;

		p = match + match_len;
	}

	/*
	 * Update the patched elf modinfo data array with namespace
	 * strings for kpatch_put_modinfo_namespaces()
	 */
	kelf_patched->modinfo_data = data;
	kelf_patched->modinfo_data_len = data_len;
}

static void kpatch_put_modinfo_namespaces(struct kpatch_elf *kelf_patched,
					  struct kpatch_elf *kelf_out)
{
	struct section *sec;

	if (!kelf_patched->modinfo_data)
		return;

	sec = create_section(kelf_out, ".modinfo",
			     kelf_patched->modinfo_data_len, 1);
	memcpy(sec->data->d_buf, kelf_patched->modinfo_data,
	       kelf_patched->modinfo_data_len);
	sec->include = 1;
}

static void kpatch_include_standard_elements(struct kpatch_elf *kelf)
{
	struct section *sec;
	struct symbol *sym;

	list_for_each_entry(sec, &kelf->sections, list) {
		/*
		 * Include the following sections even if they haven't changed.
		 *
		 * Notes about some of the more interesting sections:
		 *
		 * - With -fdata-sections, .rodata is only used for:
		 *
		 *     switch jump tables;
		 *     KASAN data (with KASAN enabled, which is rare); and
		 *     an ugly hack in vmx_vcpu_run().
		 *
		 *   Those data are all local to the functions which use them.
		 *   So it's safe to include .rodata.
		 *
		 * - On ppc64le, the .toc section is used for all data
		 *   accesses.
		 *
		 * Note that if any of these sections have rela sections, they
		 * will also be included in their entirety.  That may result in
		 * some extra (unused) klp relocations getting created, which
		 * should be harmless.
		 */
		if (!strcmp(sec->name, ".shstrtab") ||
		    !strcmp(sec->name, ".strtab") ||
		    !strcmp(sec->name, ".symtab") ||
		    !strcmp(sec->name, ".toc") ||
		    !strcmp(sec->name, ".rodata") ||
		    !strcmp(sec->name, ".rodata.str") ||
		    is_string_literal_section(sec)) {
			kpatch_include_section(sec);
		}
	}

	list_for_each_entry(sym, &kelf->symbols, list)
		if (sym->sec && is_string_literal_section(sym->sec))
			sym->include = 1;

	/* include the NULL symbol */
	list_entry(kelf->symbols.next, struct symbol, list)->include = 1;
}

static bool kpatch_include_callback_elements(struct kpatch_elf *kelf)
{
	struct section *sec;
	struct symbol *sym;
	struct rela *rela;
	bool found = false;

	/* include load/unload sections */
	list_for_each_entry(sec, &kelf->sections, list) {
		if (!is_callback_section(sec))
			continue;

		sec->include = 1;
		found = true;
		if (is_rela_section(sec)) {
			/* include callback dependencies */
			rela = list_entry(sec->relas.next, struct rela, list);
			sym = rela->sym;
			log_normal("found callback: %s\n",sym->name);
			kpatch_include_symbol(sym);
		} else if (sec->secsym) {
			sec->secsym->include = 1;
		}
	}

	/* Strip temporary structure symbols used by the callback macros. */
	list_for_each_entry(sym, &kelf->symbols, list) {
		if (sym->type == STT_OBJECT && sym->sec &&
		    is_callback_section(sym->sec))
			sym->include = 0;
	}

	return found;
}

static void kpatch_include_force_elements(struct kpatch_elf *kelf)
{
	struct section *sec;
	struct symbol *sym;
	struct rela *rela;

	/* include force sections */
	list_for_each_entry(sec, &kelf->sections, list) {
		if (!strcmp(sec->name, ".kpatch.force") ||
		    !strcmp(sec->name, ".rela.kpatch.force")) {
			sec->include = 1;
			if (!is_rela_section(sec)) {
				/* .kpatch.force */
				if (sec->secsym)
					sec->secsym->include = 1;
				continue;
			}
			/* .rela.kpatch.force */
			list_for_each_entry(rela, &sec->relas, list)
				log_normal("function '%s' marked with KPATCH_FORCE_UNSAFE!\n",
				           rela->sym->name);
		}
	}

	/* strip temporary global kpatch_force_func_* symbols */
	list_for_each_entry(sym, &kelf->symbols, list)
		if (!strncmp(sym->name, "__kpatch_force_func_",
		            strlen("__kpatch_force_func_")))
			sym->include = 0;
}

static int kpatch_include_new_globals(struct kpatch_elf *kelf)
{
	struct symbol *sym;
	int nr = 0;

	list_for_each_entry(sym, &kelf->symbols, list) {
		if (sym->bind == STB_GLOBAL && sym->sec &&
		    sym->status == NEW) {
			kpatch_include_symbol(sym);
			nr++;
		}
	}

	return nr;
}

static int kpatch_include_changed_functions(struct kpatch_elf *kelf)
{
	struct symbol *sym;
	int changed_nr = 0;

	list_for_each_entry(sym, &kelf->symbols, list) {
		if (sym->status == CHANGED &&
		    sym->type == STT_FUNC) {
			changed_nr++;
			kpatch_include_symbol(sym);
			if (sym->pfx)
				kpatch_include_symbol(sym->pfx);
		}

		if (sym->type == STT_FILE)
			sym->include = 1;
	}

	return changed_nr;
}

static void kpatch_print_changes(struct kpatch_elf *kelf)
{
	struct symbol *sym;

	list_for_each_entry(sym, &kelf->symbols, list) {
		if (!sym->include || !sym->sec || sym->type != STT_FUNC ||
		    sym->parent || sym->is_pfx)
			continue;
		if (sym->status == NEW)
			log_normal("new function: %s\n", sym->name);
		else if (sym->status == CHANGED)
			log_normal("changed function: %s\n", sym->name);
	}
}

static void kpatch_migrate_symbols(struct list_head *src,
				   struct list_head *dst,
				   bool (*select)(struct symbol *))
{
	struct symbol *sym, *safe;

	list_for_each_entry_safe(sym, safe, src, list) {
		if (select && !select(sym))
			continue;

		list_del(&sym->list);
		list_add_tail(&sym->list, dst);
	}
}

static void kpatch_migrate_included_elements(struct kpatch_elf *kelf, struct kpatch_elf **kelfout)
{
	struct section *sec, *safesec;
	struct symbol *sym, *safesym;
	struct kpatch_elf *out;

	/* allocate output kelf */
	out = malloc(sizeof(*out));
	if (!out)
		ERROR("malloc");
	memset(out, 0, sizeof(*out));
	out->arch = kelf->arch;
	out->has_pfe = kelf->has_pfe;
	out->pfe_ordered = kelf->pfe_ordered;
	INIT_LIST_HEAD(&out->sections);
	INIT_LIST_HEAD(&out->symbols);
	INIT_LIST_HEAD(&out->strings);

	/* migrate included sections from kelf to out */
	list_for_each_entry_safe(sec, safesec, &kelf->sections, list) {
		if (!sec->include)
			continue;
		list_del(&sec->list);
		list_add_tail(&sec->list, &out->sections);
		sec->index = 0;
		if (!is_rela_section(sec) && sec->secsym && !sec->secsym->include)
			/* break link to non-included section symbol */
			sec->secsym = NULL;
	}

	/* migrate included symbols from kelf to out */
	list_for_each_entry_safe(sym, safesym, &kelf->symbols, list) {
		if (!sym->include)
			continue;
		list_del(&sym->list);
		list_add_tail(&sym->list, &out->symbols);
		sym->index = 0;
		sym->strip = SYMBOL_DEFAULT;
		if (sym->sec && !sym->sec->include)
			/* break link to non-included section */
			sym->sec = NULL;

	}

	*kelfout = out;
}

static void kpatch_reorder_symbols(struct kpatch_elf *kelf)
{
	LIST_HEAD(symbols);

	/* migrate NULL sym */
	kpatch_migrate_symbols(&kelf->symbols, &symbols, is_null_sym);
	/* migrate LOCAL FILE sym */
	kpatch_migrate_symbols(&kelf->symbols, &symbols, is_file_sym);
	/* migrate LOCAL FUNC syms */
	kpatch_migrate_symbols(&kelf->symbols, &symbols, is_local_func_sym);
	/* migrate all other LOCAL syms */
	kpatch_migrate_symbols(&kelf->symbols, &symbols, is_local_sym);
	/* migrate all other (GLOBAL) syms */
	kpatch_migrate_symbols(&kelf->symbols, &symbols, NULL);

	list_replace(&symbols, &kelf->symbols);
}

static int bug_table_group_size(struct kpatch_elf *kelf, int offset)
{
	static int size = 0;
	char *str;

	if (!size) {
		str = getenv("BUG_STRUCT_SIZE");
		if (!str)
			ERROR("BUG_STRUCT_SIZE not set");
		size = atoi(str);
	}

	return size;
}

static int ex_table_group_size(struct kpatch_elf *kelf, int offset)
{
	static int size = 0;
	char *str;

	if (!size) {
		str = getenv("EX_STRUCT_SIZE");
		if (!str)
			ERROR("EX_STRUCT_SIZE not set");
		size = atoi(str);
	}

	return size;
}

static int jump_table_group_size(struct kpatch_elf *kelf, int offset)
{
	static int size = 0;
	char *str;

	if (!size) {
		str = getenv("JUMP_STRUCT_SIZE");
		if (!str)
			ERROR("JUMP_STRUCT_SIZE not set");
		size = atoi(str);
	}

	return size;
}

static int printk_index_group_size(struct kpatch_elf *kelf, int offset)
{
	static int size = 0;
	char *str;

	if (!size) {
		str = getenv("PRINTK_INDEX_STRUCT_SIZE");
		if (!str)
			ERROR("PRINTK_INDEX_STRUCT_SIZE not set");
		size = atoi(str);
	}

	return size;
}

static int parainstructions_group_size(struct kpatch_elf *kelf, int offset)
{
	static int size = 0;
	char *str;

	if (!size) {
		str = getenv("PARA_STRUCT_SIZE");
		if (!str)
			ERROR("PARA_STRUCT_SIZE not set");
		size = atoi(str);
	}

	return size;
}

static int altinstructions_group_size(struct kpatch_elf *kelf, int offset)
{
	static int size = 0;
	char *str;

	if (!size) {
		str = getenv("ALT_STRUCT_SIZE");
		if (!str)
			ERROR("ALT_STRUCT_SIZE not set");
		size = atoi(str);
	}

	return size;
}

static int smp_locks_group_size(struct kpatch_elf *kelf, int offset)
{
	return 4;
}

static int static_call_sites_group_size(struct kpatch_elf *kelf, int offset)
{
	static int size = 0;
	char *str;

	if (!size) {
		str = getenv("STATIC_CALL_STRUCT_SIZE");
		if (!str)
			ERROR("STATIC_CALL_STRUCT_SIZE not set");
		size = atoi(str);
	}

	return size;
}

static int call_sites_group_size(struct kpatch_elf *kelf, int offset)
{
	return 4;
}

static int retpoline_sites_group_size(struct kpatch_elf *kelf, int offset)
{
	return 4;
}

static int return_sites_group_size(struct kpatch_elf *kelf, int offset)
{
	return 4;
}

static int fixup_entry_group_size(struct kpatch_elf *kelf, int offset)
{
	static int size = 0;
	char *str;

	if (!size) {
		str = getenv("FIXUP_STRUCT_SIZE");
		if (!str)
			ERROR("FIXUP_STRUCT_SIZE not set");
		size = atoi(str);
	}

	return size;
}

static int fixup_lwsync_group_size(struct kpatch_elf *kelf, int offset)
{
	return 8;
}

static int fixup_barrier_nospec_group_size(struct kpatch_elf *kelf, int offset)
{
	return 8;
}

/*
 * .s390_indirect_jump, .s390_indirect_call, .s390_indirect_branches,
 * .s390_return_reg, .s390_return_mem contains indirect branch locations. This
 * is an array of 32 bit elements. These sections could be used during runtime
 * to replace the expolines with the normal indirect jump.
 */
static int s390_expolines_group_size(struct kpatch_elf *kelf, int offset)
{
	return 4;
}

/*
 * The rela groups in the .fixup section vary in size.  The beginning of each
 * .fixup rela group is referenced by the __ex_table section. To find the size
 * of a .fixup rela group, we have to traverse the __ex_table relas.
 */
static int fixup_group_size(struct kpatch_elf *kelf, int offset)
{
	struct section *relasec;
	struct rela *rela;
	int found;

	relasec = find_section_by_name(&kelf->sections, ".rela__ex_table");
	if (!relasec)
		ERROR("missing .rela__ex_table section");

	/* find beginning of this group */
	found = 0;
	list_for_each_entry(rela, &relasec->relas, list) {
		if (!strcmp(rela->sym->name, ".fixup") &&
		    rela->addend == offset) {
				found = 1;
				break;
		}
	}

	if (!found)
		ERROR("can't find .fixup rela group at offset %d\n", offset);

	/* find beginning of next group */
	found = 0;
	list_for_each_entry_continue(rela, &relasec->relas, list) {
		if (!strcmp(rela->sym->name, ".fixup") &&
		    rela->addend > offset) {
			found = 1;
			break;
		}
	}

	if (!found) {
		/* last group */
		struct section *fixupsec;
		fixupsec = find_section_by_name(&kelf->sections, ".fixup");
		if (!fixupsec)
			ERROR("missing .fixup section");
		return (int)(fixupsec->sh.sh_size - offset);
	}

	return (int)(rela->addend - offset);
}

static bool jump_table_group_filter(struct lookup_table *lookup,
				    struct section *relasec,
				    unsigned int group_offset,
				    unsigned int group_size)
{
	struct rela *code = NULL, *key = NULL, *rela;
	bool tracepoint = false, dynamic_debug = false;
	struct lookup_result symbol;
	int i = 0;

	/*
	 * Here we hard-code knowledge about the contents of the jump_entry
	 * struct.  It has three fields: code, target, and key.  Each field has
	 * a relocation associated with it.
	 */
	list_for_each_entry(rela, &relasec->relas, list) {
		if (rela->offset >= group_offset &&
		    rela->offset < group_offset + group_size) {
			if (i == 0)
				code = rela;
			else if (i == 2)
				key = rela;
			i++;
		}
	}

	if (i != 3 || !key || !code)
		ERROR("BUG: __jump_table has an unexpected format");

	if (!strncmp(key->sym->name, "__tracepoint_", 13))
		tracepoint = true;

	if (is_dynamic_debug_symbol(key->sym))
		dynamic_debug = true;

	if (KLP_ARCH) {
		/*
		 * On older kernels (with .klp.arch support), jump labels
		 * aren't supported at all.  Error out when they occur in a
		 * replacement function, with the exception of tracepoints and
		 * dynamic debug printks.  An inert tracepoint or printk is
		 * harmless enough, but a broken jump label can cause
		 * unexpected behavior.
		 */
		if (tracepoint || dynamic_debug)
			return false;

		/*
		 * This will be upgraded to an error after all jump labels have
		 * been reported.
		 */
		log_error("Found a jump label at %s()+0x%lx, using key %s.  Jump labels aren't supported with this kernel.  Use static_key_enabled() instead.\n",
			  code->sym->name, code->addend, key->sym->name);
		jump_label_errors++;
		return false;
	}

	/*
	 * On newer (5.8+) kernels, jump labels are supported in the case where
	 * the corresponding static key lives in vmlinux.  That's because such
	 * kernels apply vmlinux-specific .klp.rela sections at the same time
	 * (in the klp module load) as normal relas, before jump label init.
	 * On the other hand, jump labels based on static keys which are
	 * defined in modules aren't supported, because late module patching
	 * can result in the klp relocations getting applied *after* the klp
	 * module's jump label init.
	 */

	if (lookup_symbol(lookup, key->sym, &symbol) &&
	    strcmp(symbol.objname, "vmlinux")) {

		/* The static key lives in a module -- not supported */

		/* Inert tracepoints and dynamic debug printks are harmless */
		if (tracepoint || dynamic_debug)
			return false;

		/*
		 * This will be upgraded to an error after all jump label
		 * errors have been reported.
		 */
		log_error("Found a jump label at %s()+0x%lx, using key %s, which is defined in a module.  Use static_key_enabled() instead.\n",
			  code->sym->name, code->addend, key->sym->name);
		jump_label_errors++;
		return false;
	}

	/* The static key lives in vmlinux or the patch module itself */

	/*
	 * If the jump label key lives in the '__dyndbg' section, make sure
	 * the section gets included, because we don't use klp relocs for
	 * dynamic debug symbols.  For an example of such a key, see
	 * DYNAMIC_DEBUG_BRANCH().
	 */
	if (dynamic_debug)
		kpatch_include_symbol(key->sym);

	return true;
}

static bool static_call_sites_group_filter(struct lookup_table *lookup,
					   struct section *relasec,
					   unsigned int group_offset,
					   unsigned int group_size)
{
	struct rela *code = NULL, *key = NULL, *rela;
	bool tracepoint = false;
	struct lookup_result symbol;
	int i = 0;

	/*
	 * Here we hard-code knowledge about the contents of the jump_entry
	 * struct.  It has three fields: code, target, and key.  Each field has
	 * a relocation associated with it.
	 */
	list_for_each_entry(rela, &relasec->relas, list) {
		if (rela->offset >= group_offset &&
		    rela->offset < group_offset + group_size) {
			if (i == 0)
				code = rela;
			else if (i == 1)
				key = rela;
			i++;
		}
	}

	if (i != 2 || !key || !code)
		ERROR("BUG: .static_call_sites has an unexpected format");

	if (!strncmp(key->sym->name, "__SCK__tp_func_", 15))
		tracepoint = true;

	/*
	 * Static calls are only supported in the case where the corresponding
	 * static call key lives in vmlinux (see explanation in
	 * jump_table_group_filter).
	 */

	if (lookup_symbol(lookup, key->sym, &symbol) &&
	    strcmp(symbol.objname, "vmlinux")) {

		/* The key lives in a module -- not supported */

		/* Inert tracepoints are harmless */
		if (tracepoint)
			return false;

		/*
		 * This will be upgraded to an error after all static call
		 * errors have been reported.
		 */
		log_error("Found a static call at %s()+0x%lx, using key %s, which is defined in a module.  Use KPATCH_STATIC_CALL() instead.\n",
			   code->sym->name, code->addend, key->sym->name);
		static_call_errors++;
		return false;
	}

	/* The key lives in vmlinux or the patch module itself */
	return true;
}


static struct special_section special_sections[] = {
	{
		.name		= "__bug_table",
		.arch		= AARCH64 | X86_64 | PPC64 | S390,
		.group_size	= bug_table_group_size,
	},
	{
		.name		= ".fixup",
		.arch		= AARCH64 | X86_64 | PPC64 | S390,
		.group_size	= fixup_group_size,
	},
	{
		.name		= "__ex_table", /* must come after .fixup */
		.arch		= AARCH64 | X86_64 | PPC64 | S390,
		.group_size	= ex_table_group_size,
	},
	{
		.name		= "__jump_table",
		.arch		= AARCH64 | X86_64 | PPC64 | S390,
		.group_size	= jump_table_group_size,
		.group_filter	= jump_table_group_filter,
	},
	{
		.name		= ".printk_index",
		.arch		= AARCH64 | X86_64 | PPC64 | S390,
		.group_size	= printk_index_group_size,
	},
	{
		.name		= ".smp_locks",
		.arch		= X86_64,
		.group_size	= smp_locks_group_size,
	},
	{
		.name		= ".parainstructions",
		.arch		= X86_64,
		.group_size	= parainstructions_group_size,
	},
	{
		.name		= ".altinstructions",
		.arch		= AARCH64 | X86_64 | S390,
		.group_size	= altinstructions_group_size,
	},
	{
		.name		= ".static_call_sites",
		.arch		= X86_64,
		.group_size	= static_call_sites_group_size,
		.group_filter	= static_call_sites_group_filter,
	},
	{
		.name		= ".call_sites",
		.arch		= X86_64,
		.group_size	= call_sites_group_size,
	},
	{
		.name		= ".retpoline_sites",
		.arch		= X86_64,
		.group_size	= retpoline_sites_group_size,
	},
	{
		.name		= ".return_sites",
		.arch		= X86_64,
		.group_size	= return_sites_group_size,
	},
	{
		.name		= "__ftr_fixup",
		.arch		= PPC64,
		.group_size	= fixup_entry_group_size,
	},
	{
		.name		= "__mmu_ftr_fixup",
		.arch		= PPC64,
		.group_size	= fixup_entry_group_size,
	},
	{
		.name		= "__fw_ftr_fixup",
		.arch		= PPC64,
		.group_size	= fixup_entry_group_size,
	},
	{
		.name		= "__lwsync_fixup",
		.arch		= PPC64,
		.group_size	= fixup_lwsync_group_size,
	},
	{
		.name		= "__barrier_nospec_fixup",
		.arch		= PPC64,
		.group_size	= fixup_barrier_nospec_group_size,
	},
	{
		.name		= ".s390_return_mem",
		.arch		= S390,
		.group_size	= s390_expolines_group_size,
	},
	{
		.name		= ".s390_return_reg",
		.arch		= S390,
		.group_size	= s390_expolines_group_size,
	},
	{
		.name		= ".s390_indirect_call",
		.arch		= S390,
		.group_size	= s390_expolines_group_size,
	},
	{
		.name		= ".s390_indirect_branches",
		.arch		= S390,
		.group_size	= s390_expolines_group_size,
	},
	{
		.name		= ".s390_indirect_jump",
		.arch		= S390,
		.group_size	= s390_expolines_group_size,
	},
	{},
};

static bool should_keep_rela_group(struct lookup_table *lookup,
				   struct section *relasec, unsigned int offset,
				   unsigned int size)
{
	struct rela *rela;
	bool found = false;

	/* check if any relas in the group reference any changed functions */
	list_for_each_entry(rela, &relasec->relas, list) {
		if (rela->offset >= offset &&
		    rela->offset < offset + size &&
		    rela->sym->type == STT_FUNC &&
		    rela->sym->sec->include) {
			found = true;
			log_debug("new/changed symbol %s found in special section %s\n",
				  rela->sym->name, relasec->name);
		}
	}

	if (!found)
		return false;

	return true;
}

/*
 * When updating .fixup, the corresponding addends in .ex_table need to be
 * updated too. Stash the result in rela.r_addend so that the calculation in
 * fixup_group_size() is not affected.
 */
static void kpatch_update_ex_table_addend(struct kpatch_elf *kelf,
					  struct special_section *special,
					  int src_offset, int dest_offset,
					  int group_size)
{
	struct rela *rela;
	struct section *relasec;

	relasec = find_section_by_name(&kelf->sections, ".rela__ex_table");
	if (!relasec)
		ERROR("missing .rela__ex_table section");

	list_for_each_entry(rela, &relasec->relas, list) {
		if (!strcmp(rela->sym->name, ".fixup") &&
		    rela->addend >= src_offset &&
		    rela->addend < src_offset + group_size)
			rela->rela.r_addend = rela->addend - (src_offset - dest_offset);
	}
}

static void kpatch_regenerate_special_section(struct kpatch_elf *kelf,
					      struct lookup_table *lookup,
					      struct special_section *special,
					      struct section *relasec)
{
	struct rela *rela, *safe;
	char *src, *dest;
	unsigned int group_size, src_offset, dest_offset;

	LIST_HEAD(newrelas);

	src = relasec->base->data->d_buf;
	/* alloc buffer for new base section */
	dest = malloc(relasec->base->sh.sh_size);
	if (!dest)
		ERROR("malloc");

	/* Restore the stashed r_addend from kpatch_update_ex_table_addend. */
	if (!strcmp(special->name, "__ex_table")) {
		list_for_each_entry(rela, &relasec->relas, list) {
			if (!strcmp(rela->sym->name, ".fixup"))
				rela->addend = rela->rela.r_addend;
		}
	}

	src_offset = 0;
	dest_offset = 0;
	for ( ; src_offset < relasec->base->sh.sh_size; src_offset += group_size) {

		group_size = special->group_size(kelf, src_offset);

		/*
		 * In some cases the struct has padding at the end to ensure
		 * that all structs after it are properly aligned.  But the
		 * last struct in the section may not be padded.  In that case,
		 * shrink the group_size such that it still (hopefully)
		 * contains the data but doesn't go past the end of the
		 * section.
		 */
		if (src_offset + group_size > relasec->base->sh.sh_size)
			group_size = (unsigned int)(relasec->base->sh.sh_size - src_offset);

		if (!should_keep_rela_group(lookup, relasec, src_offset, group_size))
			continue;

		if (special->group_filter &&
		    !special->group_filter(lookup, relasec, src_offset, group_size))
			continue;

		/*
		 * Copy all relas in the group.  It's possible that the relas
		 * aren't sorted (e.g. .rela.fixup), so go through the entire
		 * rela list each time.
		 */
		list_for_each_entry_safe(rela, safe, &relasec->relas, list) {
			if (rela->offset >= src_offset &&
			    rela->offset < src_offset + group_size) {
				/* copy rela entry */
				list_del(&rela->list);
				list_add_tail(&rela->list, &newrelas);

				rela->offset -= src_offset - dest_offset;
				rela->rela.r_offset = rela->offset;

				rela->sym->include = 1;

				if (!strcmp(special->name, ".fixup"))
					kpatch_update_ex_table_addend(kelf, special,
								      src_offset,
								      dest_offset,
								      group_size);
			}
		}

		/* copy base section group */
		memcpy(dest + dest_offset, src + src_offset, group_size);
		dest_offset += group_size;
	}

	if (jump_label_errors)
		ERROR("Found %d unsupported jump label(s) in the patched code. Use static_key_enabled() instead.",
		      jump_label_errors);

	if (static_call_errors)
		ERROR("Found %d unsupported static call(s) in the patched code. Use KPATCH_STATIC_CALL() instead.",
		      static_call_errors);

	if (!dest_offset) {
		/* no changed or global functions referenced */
		relasec->status = relasec->base->status = SAME;
		relasec->include = relasec->base->include = 0;
		free(dest);
		return;
	}

	/* overwrite with new relas list */
	list_replace(&newrelas, &relasec->relas);

	/* include both rela and base sections */
	relasec->include = 1;
	relasec->base->include = 1;
	/* include secsym so .kpatch.arch relas can point to section symbols */
	if (relasec->base->secsym)
		relasec->base->secsym->include = 1;

	/*
	 * Update text section data buf and size.
	 *
	 * The rela section's data buf and size will be regenerated in
	 * kpatch_rebuild_rela_section_data().
	 */
	relasec->base->data->d_buf = dest;
	relasec->base->data->d_size = dest_offset;
}

#define ORC_IP_PTR_SIZE 4

/*
 * This function is similar to kpatch_regenerate_special_section(), but
 * customized for the ORC-related sections.  ORC is more special than the other
 * special sections because each ORC entry is split into .orc_unwind (struct
 * orc_entry) and .orc_unwind_ip.
 */
static void kpatch_regenerate_orc_sections(struct kpatch_elf *kelf)
{
	struct rela *rela, *safe;
	char *src, *dest, *str;
	unsigned int src_idx = 0, dest_idx = 0, orc_entry_size;
	struct section *orc_sec, *ip_sec;


	str = getenv("ORC_STRUCT_SIZE");
	if (!str)
		return;
	orc_entry_size = atoi(str);

	if (!orc_entry_size)
		ERROR("bad ORC_STRUCT_SIZE");

	LIST_HEAD(newrelas);

	orc_sec = find_section_by_name(&kelf->sections, ".orc_unwind");
	ip_sec  = find_section_by_name(&kelf->sections, ".orc_unwind_ip");

	if (!orc_sec || !ip_sec)
		return;

	if (orc_sec->sh.sh_size % orc_entry_size != 0)
		ERROR("bad .orc_unwind size");

	if (ip_sec->sh.sh_size !=
	    (orc_sec->sh.sh_size / orc_entry_size) * ORC_IP_PTR_SIZE)
		ERROR(".orc_unwind/.orc_unwind_ip size mismatch");

	src = orc_sec->data->d_buf;
	dest = malloc(orc_sec->sh.sh_size);
	if (!dest)
		ERROR("malloc");

	list_for_each_entry_safe(rela, safe, &ip_sec->rela->relas, list) {

		if (rela->sym->type != STT_FUNC || !rela->sym->sec->include)
			goto next;

		/* copy orc entry */
		memcpy(dest + (dest_idx * orc_entry_size),
		       src + (src_idx * orc_entry_size),
		       orc_entry_size);

		/* move ip rela */
		list_del(&rela->list);
		list_add_tail(&rela->list, &newrelas);
		rela->offset = dest_idx * ORC_IP_PTR_SIZE;
		rela->sym->include = 1;

		dest_idx++;
next:
		src_idx++;
	}

	if (!dest_idx) {
		/* no changed or global functions referenced */
		orc_sec->status = ip_sec->status = ip_sec->rela->status = SAME;
		orc_sec->include = ip_sec->include = ip_sec->rela->include = 0;
		free(dest);
		return;
	}

	/* overwrite with new relas list */
	list_replace(&newrelas, &ip_sec->rela->relas);

	/* include the sections */
	orc_sec->include = ip_sec->include = ip_sec->rela->include = 1;

	/*
	 * Update data buf/size.
	 *
	 * The ip section can keep its old (zeroed data), though its size has
	 * possibly decreased.  The ip rela section's data buf and size will be
	 * regenerated in kpatch_rebuild_rela_section_data().
	 */
	orc_sec->data->d_buf = dest;
	orc_sec->data->d_size = dest_idx * orc_entry_size;
	ip_sec->data->d_size = dest_idx * ORC_IP_PTR_SIZE;
}

static void kpatch_check_relocations(struct kpatch_elf *kelf)
{
	struct rela *rela;
	struct section *relasec;
	long sec_size;
	long sec_off;

	list_for_each_entry(relasec, &kelf->sections, list) {
		if (!is_rela_section(relasec))
			continue;
		list_for_each_entry(rela, &relasec->relas, list) {
			if (!rela->sym->sec)
				continue;

			sec_size = rela->sym->sec->data->d_size;
			sec_off = (long)rela->sym->sym.st_value +
				  rela_target_offset(kelf, relasec, rela);

			/*
			 * This check isn't perfect: we still allow relocations
			 * to the end of a section.  There are real instances
			 * of that, including ORC entries, LOCKDEP=n
			 * zero-length '__key' passing, and the loop edge case
			 * described in kpatch_replace_sections_syms().  For
			 * now, just allow all such cases.
			 */
			if (sec_off < 0 || sec_off > sec_size) {
				ERROR("%s+0x%x: out-of-range relocation %s+%lx",
				      relasec->base->name, rela->offset,
				      rela->sym->name, rela->addend);
			}
		}
	}
}

static void kpatch_include_debug_sections(struct kpatch_elf *kelf)
{
	struct section *sec;
	struct rela *rela, *saferela;

	/* include all .debug_* sections */
	list_for_each_entry(sec, &kelf->sections, list) {
		if (is_debug_section(sec)) {
			sec->include = 1;
			if (!is_rela_section(sec) && sec->secsym)
				sec->secsym->include = 1;
		}
	}

	/*
	 * Go through the .rela.debug_ sections and strip entries
	 * referencing unchanged symbols
	 */
	list_for_each_entry(sec, &kelf->sections, list) {
		if (!is_rela_section(sec) || !is_debug_section(sec))
			continue;
		list_for_each_entry_safe(rela, saferela, &sec->relas, list)
			if (!rela->sym->sec->include)
				list_del(&rela->list);
	}
}

static void kpatch_mark_ignored_sections(struct kpatch_elf *kelf)
{
	struct section *sec, *strsec, *ignoresec;
	struct rela *rela;
	char *name;

	/* Ignore any discarded sections */
	list_for_each_entry(sec, &kelf->sections, list) {
		if (!strncmp(sec->name, ".discard", 8) ||
		    !strncmp(sec->name, ".rela.discard", 13) ||
		    !strncmp(sec->name, ".llvm_addrsig", 13) ||
		    !strncmp(sec->name, ".llvm.", 6))
			sec->ignore = 1;

		/*
		 * Ignore .modinfo changes, but do stash any module
		 * namespaces for kpatch_put_modinfo_namespaces()
		 */
		if (!strcmp(sec->name, ".modinfo")) {
			kpatch_get_modinfo_namespaces(kelf, sec);
			sec->ignore = 1;
		}

		if (kelf->arch == X86_64) {
			if (!strcmp(sec->name, ".rela__patchable_function_entries") ||
			    !strcmp(sec->name, "__patchable_function_entries"))
				sec->ignore = 1;
		}
	}

	sec = find_section_by_name(&kelf->sections, ".kpatch.ignore.sections");
	if (!sec)
		return;

	list_for_each_entry(rela, &sec->rela->relas, list) {
		strsec = rela->sym->sec;
		strsec->status = CHANGED;
		/*
		 * Include the string section here.  This is because the
		 * KPATCH_IGNORE_SECTION() macro is passed a literal string
		 * by the patch author, resulting in a change to the string
		 * section.  If we don't include it, then we will potentially
		 * get a "changed section not included" error in
		 * kpatch_verify_patchability() if no other function based change
		 * also changes the string section.  We could try to exclude each
		 * literal string added to the section by KPATCH_IGNORE_SECTION()
		 * from the section data comparison, but this is a simpler way.
		 */
		strsec->include = 1;
		if (strsec->secsym)
			strsec->secsym->include = 1;

		name = strsec->data->d_buf + rela->addend;
		ignoresec = find_section_by_name(&kelf->sections, name);
		if (!ignoresec)
			ERROR("KPATCH_IGNORE_SECTION: can't find %s", name);
		log_normal("ignoring section: %s\n", name);
		if (is_rela_section(ignoresec))
			ignoresec = ignoresec->base;
		ignoresec->ignore = 1;
		if (ignoresec->twin)
			ignoresec->twin->ignore = 1;
	}
}

static void kpatch_mark_ignored_sections_same(struct kpatch_elf *kelf)
{
	struct section *sec;
	struct symbol *sym;

	list_for_each_entry(sec, &kelf->sections, list) {
		if (!sec->ignore)
			continue;
		sec->status = SAME;
		if (!is_rela_section(sec)) {
			if (sec->secsym)
				sec->secsym->status = SAME;
			if (sec->rela)
				sec->rela->status = SAME;
		}
		list_for_each_entry(sym, &kelf->symbols, list) {
			if (sym->sec != sec)
				continue;
			sym->status = SAME;
		}
	}

	/* strip temporary global __UNIQUE_ID_kpatch_ignore_section_* symbols */
	list_for_each_entry(sym, &kelf->symbols, list)
		if (!strncmp(sym->name, "__UNIQUE_ID_kpatch_ignore_section_",
		            strlen("__UNIQUE_ID_kpatch_ignore_section_")))
			sym->status = SAME;
}

static void kpatch_mark_ignored_children_same(struct symbol *sym)
{
	struct symbol *child;

	list_for_each_entry(child, &sym->children, subfunction_node) {
		child->status = SAME;
		kpatch_mark_ignored_children_same(child);
	}
}

static void kpatch_mark_ignored_functions_same(struct kpatch_elf *kelf)
{
	struct section *sec;
	struct symbol *sym;
	struct rela *rela;

	sec = find_section_by_name(&kelf->sections, ".kpatch.ignore.functions");
	if (!sec)
		return;

	list_for_each_entry(rela, &sec->rela->relas, list) {
		if (!rela->sym->sec)
			ERROR("expected bundled symbol");
		if (rela->sym->type != STT_FUNC)
			ERROR("expected function symbol");
		log_normal("ignoring function: %s\n", rela->sym->name);
		if (rela->sym->status != CHANGED)
			log_normal("NOTICE: no change detected in function %s, unnecessary KPATCH_IGNORE_FUNCTION()?\n", rela->sym->name);
		rela->sym->status = SAME;
		rela->sym->sec->status = SAME;

		kpatch_mark_ignored_children_same(rela->sym);

		if (rela->sym->sec->secsym)
			rela->sym->sec->secsym->status = SAME;
		if (rela->sym->sec->rela)
			rela->sym->sec->rela->status = SAME;
	}

	/* strip temporary global kpatch_ignore_func_* symbols */
	list_for_each_entry(sym, &kelf->symbols, list)
		if (!strncmp(sym->name, "__kpatch_ignore_func_",
		            strlen("__kpatch_ignore_func_")))
			sym->status = SAME;
}

static void kpatch_create_kpatch_arch_section(struct kpatch_elf *kelf, char *objname)
{
	struct special_section *special;
	struct symbol *strsym;
	struct section *sec, *karch_sec;
	struct rela *rela;
	int nr, index = 0;

	if (!KLP_ARCH)
		return;

	nr = sizeof(special_sections) / sizeof(special_sections[0]);
	karch_sec = create_section_pair(kelf, ".kpatch.arch", sizeof(struct kpatch_arch), nr);

	/* lookup strings symbol */
	strsym = find_symbol_by_name(&kelf->symbols, ".kpatch.strings");
	if (!strsym)
		ERROR("can't find .kpatch.strings symbol");

	for (special = special_sections; special->name; special++) {
		if ((special->arch & kelf->arch) == 0)
			continue;

		if (strcmp(special->name, ".parainstructions") &&
		    strcmp(special->name, ".altinstructions"))
			continue;

		sec = find_section_by_name(&kelf->sections, special->name);
		if (!sec)
			continue;

		/* entries[index].sec */
		ALLOC_LINK(rela, &karch_sec->rela->relas);
		rela->sym = sec->secsym;
		rela->type = absolute_rela_type(kelf);
		rela->addend = 0;
		rela->offset = (unsigned int)(index * sizeof(struct kpatch_arch) + \
			       offsetof(struct kpatch_arch, sec));

		/* entries[index].objname */
		ALLOC_LINK(rela, &karch_sec->rela->relas);
		rela->sym = strsym;
		rela->type = absolute_rela_type(kelf);
		rela->addend = offset_of_string(&kelf->strings, objname);
		rela->offset = (unsigned int)(index * sizeof(struct kpatch_arch) + \
			       offsetof(struct kpatch_arch, objname));

		index++;
	}

	karch_sec->data->d_size = index * sizeof(struct kpatch_arch);
	karch_sec->sh.sh_size = karch_sec->data->d_size;
}

static void kpatch_process_special_sections(struct kpatch_elf *kelf,
					    struct lookup_table *lookup)
{
	struct special_section *special;
	struct section *sec;
	struct symbol *sym;
	struct rela *rela;
	int altinstr = 0;

	for (special = special_sections; special->name; special++) {
		if ((special->arch & kelf->arch) == 0)
			continue;

		sec = find_section_by_name(&kelf->sections, special->name);
		if (!sec || !sec->rela)
			continue;

		kpatch_regenerate_special_section(kelf, lookup, special, sec->rela);

		if (!strcmp(special->name, ".altinstructions") && sec->include)
			altinstr = 1;
	}

	/*
	 * The following special sections don't have relas which reference
	 * non-included symbols, so their entire rela section can be included.
	 */
	list_for_each_entry(sec, &kelf->sections, list) {
		if (strcmp(sec->name, ".altinstr_replacement"))
			continue;
		/*
		 * Only include .altinstr_replacement if .altinstructions
		 * is also included.
		 */
		if (!altinstr)
			break;

		/* include base section */
		sec->include = 1;

		/* include all symbols in the section */
		list_for_each_entry(sym, &kelf->symbols, list)
			if (sym->sec == sec)
				sym->include = 1;

		/* include rela section */
		if (sec->rela) {
			sec->rela->include = 1;
			/* include all symbols referenced by relas */
			list_for_each_entry(rela, &sec->rela->relas, list)
				kpatch_include_symbol(rela->sym);
		}
	}

	if (KLP_ARCH) {
		/*
		 * The following special sections aren't supported with older
		 * kernels, so make sure we don't ever try to include them.
		 * Otherwise the kernel will see the jump table during module
		 * loading and get confused.  Generally it should be safe to
		 * exclude them, it just means that you can't modify jump
		 * labels and enable tracepoints in a patched function.
		 */
		list_for_each_entry(sec, &kelf->sections, list) {
			if (strcmp(sec->name, "__jump_table") &&
			    strcmp(sec->name, "__tracepoints") &&
			    strcmp(sec->name, "__tracepoints_ptrs") &&
			    strcmp(sec->name, "__tracepoints_strings"))
				continue;

			sec->status = SAME;
			sec->include = 0;
			if (sec->rela) {
				sec->rela->status = SAME;
				sec->rela->include = 0;
			}
		}
	}

	kpatch_regenerate_orc_sections(kelf);
}

static void kpatch_create_patches_sections(struct kpatch_elf *kelf,
					   struct lookup_table *table,
					   char *objname)
{
	int nr, index, objname_offset;
	struct section *sec, *relasec;
	struct symbol *sym, *strsym;
	struct rela *rela;
	struct lookup_result symbol;
	struct kpatch_patch_func *funcs;

	/* count patched functions */
	nr = 0;
	list_for_each_entry(sym, &kelf->symbols, list) {
		if (sym->type != STT_FUNC || sym->status != CHANGED ||
		    sym->parent)
			continue;
		nr++;
	}

	/* create text/rela section pair */
	sec = create_section_pair(kelf, ".kpatch.funcs", sizeof(*funcs), nr);
	relasec = sec->rela;
	funcs = sec->data->d_buf;

	/* lookup strings symbol */
	strsym = find_symbol_by_name(&kelf->symbols, ".kpatch.strings");
	if (!strsym)
		ERROR("can't find .kpatch.strings symbol");

	/* add objname to strings */
	objname_offset = offset_of_string(&kelf->strings, objname);

	/* populate sections */
	index = 0;
	list_for_each_entry(sym, &kelf->symbols, list) {
		if (sym->type != STT_FUNC || sym->status != CHANGED ||
		    sym->parent)
			continue;

		if (!lookup_symbol(table, sym, &symbol))
			ERROR("can't find symbol '%s' in symbol table", sym->name);

		if (sym->bind == STB_LOCAL && symbol.global)
			ERROR("can't find local symbol '%s' in symbol table", sym->name);

		log_debug("lookup for %s: obj=%s sympos=%lu size=%lu",
			  sym->name, symbol.objname, symbol.sympos,
			  symbol.size);

		/*
		 * Convert global symbols to local so other objects in the
		 * patch module (like the patch callback object's init code)
		 * won't link to this function and call it before its
		 * relocations have been applied.
		 */
		sym->bind = STB_LOCAL;
		sym->sym.st_info = (unsigned char)
				   GELF_ST_INFO(sym->bind, sym->type);

		/* add entry in text section */
		funcs[index].old_addr = symbol.addr;
		funcs[index].old_size = symbol.size;
		funcs[index].new_size = sym->sym.st_size;
		funcs[index].sympos = symbol.sympos;

		/*
		 * Add a relocation that will populate the
		 * funcs[index].new_addr field at module load time.
		 */
		ALLOC_LINK(rela, &relasec->relas);
		rela->sym = sym;
		rela->type = absolute_rela_type(kelf);
		rela->addend = 0;
		rela->offset = (unsigned int)(index * sizeof(*funcs));

		/*
		 * Add a relocation that will populate the funcs[index].name
		 * field.
		 */
		ALLOC_LINK(rela, &relasec->relas);
		rela->sym = strsym;
		rela->type = absolute_rela_type(kelf);
		rela->addend = offset_of_string(&kelf->strings, sym->name);
		rela->offset = (unsigned int)(index * sizeof(*funcs) +
			       offsetof(struct kpatch_patch_func, name));

		/*
		 * Add a relocation that will populate the funcs[index].objname
		 * field.
		 */
		ALLOC_LINK(rela, &relasec->relas);
		rela->sym = strsym;
		rela->type = absolute_rela_type(kelf);
		rela->addend = objname_offset;
		rela->offset = (unsigned int)(index * sizeof(*funcs) +
			       offsetof(struct kpatch_patch_func,objname));

		index++;
	}

	/* sanity check, index should equal nr */
	if (index != nr)
		ERROR("size mismatch in funcs sections");

}

static bool kpatch_is_core_module_symbol(char *name)
{
	return (!strcmp(name, "kpatch_shadow_alloc") ||
		!strcmp(name, "kpatch_shadow_free") ||
		!strcmp(name, "kpatch_shadow_get"));
}

static bool is_expoline(struct kpatch_elf *kelf, char *name)
{
	return kelf->arch == S390 && !strncmp(name, "__s390_indirect_jump_r", 22);
}

static int function_ptr_rela(const struct rela *rela, struct kpatch_elf *kelf)
{
	const struct rela *rela_toc = toc_rela(rela);
	bool funcptr = false;

	if (rela_toc && rela_toc->sym->type == STT_FUNC && !rela_toc->sym->parent) {
		switch (kelf->arch) {
		case X86_64:
			if (rela_toc->addend == (int)rela_toc->sym->sym.st_value &&
			    rela->type == R_X86_64_32S)
				funcptr = true;
			break;
		case PPC64:
			if (rela_toc->addend == (int)rela_toc->sym->sym.st_value &&
			    (rela->type == R_PPC64_TOC16_HA || rela->type == R_PPC64_TOC16_LO_DS))
				funcptr = true;
			break;
		case S390:
			if (rela->type == R_390_GOTENT)
				funcptr = true;
			break;
		default:
			break;
		}
	}
	return funcptr;
}

static bool need_klp_reloc(struct kpatch_elf *kelf, struct lookup_table *table,
			   struct section *relasec, const struct rela *rela)
{
	struct lookup_result symbol;

	if (is_debug_section(relasec))
		return false;

	/*
	 * These references are treated specially by the module loader and
	 * should never be converted to klp relocations.
	 */
	if (rela->type == R_PPC64_REL16_HA || rela->type == R_PPC64_REL16_LO ||
	    rela->type == R_PPC64_ENTRY)
		return false;

	/* v5.13+ kernels use relative jump labels */
	if (rela->type == R_PPC64_REL64 && strcmp(relasec->name, ".rela__jump_table"))
		return false;

	/*
	 * On powerpc, the function prologue generated by GCC 6 has the
	 * sequence:
	 *
	 *	.globl my_func
	 *	.type my_func, @function
	 *	.quad .TOC.-my_func
	 * my_func:
	 *	.reloc ., R_PPC64_ENTRY ; optional
	 *	ld r2,-8(r12)
	 *	add r2,r2,r12
	 *	.localentry my_func, .-my_func
	 *
	 * The R_PPC64_ENTRY is optional and its symbol might have an empty
	 * name.  Leave it as a normal rela.
	 */
	if (rela->type == R_PPC64_ENTRY)
		return false;

	/*
	 * Allow references to core module symbols to remain as normal
	 * relas.  They should be exported.
	 */
	if (kpatch_is_core_module_symbol(rela->sym->name))
		return false;

       /*
        * Allow references to s390 expolines to remain as normal relas.  They
        * will be generated in the module by the kernel module link.
        */
	if (is_expoline(kelf, rela->sym->name))
		return false;

	if (rela->sym->sec) {
		/*
		 * Internal symbols usually don't need klp relocations, because
		 * they live in the patch module and can be relocated normally.
		 *
		 * There's one exception: function pointers.
		 *
		 * If the rela references a function pointer, we convert it to
		 * a klp relocation, so that the function pointer will refer to
		 * the original function rather than the patched function.
		 * This can prevent crashes in cases where the function pointer
		 * is called asynchronously after the patch module has been
		 * unloaded.
		 */
		if (!function_ptr_rela(rela, kelf))
			return false;

		/*
		 * Function pointers which refer to _nested_ functions are a
		 * special case.  They are not supposed to be visible outside
		 * of the function that defines them.  Their names may differ
		 * in the original and the patched kernels which makes it
		 * difficult to use klp relocations.  Fortunately, nested
		 * functions are rare and are unlikely to be used as
		 * asynchronous callbacks, so the patched code can refer to
		 * them directly.  It seems, one can only distinguish such
		 * functions by their names containing a dot.  Other kinds of
		 * functions with such names (e.g. optimized copies of
		 * functions) are unlikely to be used as callbacks.
		 *
		 * Function pointers to *new* functions don't have this issue,
		 * just use a normal rela for them.
		 */
		return toc_rela(rela)->sym->status != NEW &&
			!strchr(toc_rela(rela)->sym->name, '.');
	}

	if (!lookup_symbol(table, rela->sym, &symbol)) {
		/*
		 * Assume the symbol lives in another .o in the patch module.
		 * A normal rela should work.
		 */
		return false;
	}

	if (rela->sym->bind == STB_LOCAL) {

		if (symbol.global)
			ERROR("can't find local symbol '%s' in symbol table",
			      rela->sym->name);

		/*
		 * The symbol is (formerly) local.  Use a klp relocation to
		 * access the original version of the symbol in the patched
		 * object.
		 */
		return true;
	}

	if (symbol.exported) {

		if (is_gcc6_localentry_bundled_sym(kelf, rela->sym)) {
			/*
			 * On powerpc, the symbol is global and exported, but
			 * it was also in the changed object file.  In this
			 * case the rela refers to the 'localentry' point, so a
			 * normal rela wouldn't work.  Force a klp relocation
			 * so it can be handled correctly by the livepatch
			 * relocation code.
			 */
			return true;
		}

		if (!strcmp(symbol.objname, "vmlinux")) {
			/*
			 * The symbol is exported by vmlinux.  Use a normal
			 * rela.
			 */
			return false;
		}

		/*
		 * The symbol is exported by the to-be-patched module, or by
		 * another module which the patched module depends on.  Use a
		 * klp relocation because of late module loading: the patch
		 * module may be loaded before the to-be-patched (or other)
		 * module.
		 */
		return true;
	}

	if (symbol.global) {
		/*
		 * The symbol is global in the to-be-patched object, but not
		 * exported.  Use a klp relocation to work around the fact that
		 * it's an unexported sybmbol.
		 */
		return true;
	}

	/*
	 * The symbol is global and not exported, but it's not in the parent
	 * object.  The only explanation is that it's defined in another object
	 * in the patch module.  A normal rela should resolve it.
	 */
	return false;
}

/*
 * kpatch_create_intermediate_sections()
 *
 * The primary purpose of this function is to convert some relocations to klp
 * relocations.
 *
 * If the patched code refers to a symbol, for example, if it calls a function
 * or stores a pointer to a function somewhere or accesses some global data,
 * the address of that symbol must be resolved somehow before the patch is
 * applied.
 *
 * If the symbol lives outside the patch module, and if it's not exported by
 * vmlinux (e.g., with EXPORT_SYMBOL) then the rela needs to be converted to a
 * klp relocation so the livepatch code can resolve it at runtime.
 */
static void kpatch_create_intermediate_sections(struct kpatch_elf *kelf,
						struct lookup_table *table,
						char *objname,
						char *pmod_name)
{
	int nr, index;
	struct section *relasec, *ksym_sec, *krela_sec;
	struct rela *rela, *rela2, *safe;
	struct symbol *strsym, *ksym_sec_sym;
	struct kpatch_symbol *ksyms;
	struct kpatch_relocation *krelas;
	struct lookup_result symbol;
	bool special;
	bool vmlinux = !strcmp(objname, "vmlinux");
	struct special_section *s;

	/* count rela entries that need to be dynamic */
	nr = 0;
	list_for_each_entry(relasec, &kelf->sections, list) {
		if (!is_rela_section(relasec))
			continue;
		if (!strcmp(relasec->name, ".rela.kpatch.funcs"))
			continue;
		list_for_each_entry(rela, &relasec->relas, list) {

			/* upper bound on number of kpatch relas and symbols */
			nr++;

			/*
			 * We set 'need_klp_reloc' here in the first pass
			 * because the .toc section's 'need_klp_reloc' values
			 * are dependent on all the other sections.  Otherwise,
			 * if we did this analysis in the second pass, we'd
			 * have to convert .toc klp relocations at the very end.
			 *
			 * Specifically, this is needed for the powerpc
			 * internal symbol function pointer check which is done
			 * via .toc indirection in need_klp_reloc().
			 */
			if (need_klp_reloc(kelf, table, relasec, rela))
				toc_rela(rela)->need_klp_reloc = true;
		}
	}

	/* create .kpatch.relocations text/rela section pair */
	krela_sec = create_section_pair(kelf, ".kpatch.relocations", sizeof(*krelas), nr);
	krelas = krela_sec->data->d_buf;

	/* create .kpatch.symbols text/rela section pair */
	ksym_sec = create_section_pair(kelf, ".kpatch.symbols", sizeof(*ksyms), nr);
	ksyms = ksym_sec->data->d_buf;

	/* create .kpatch.symbols section symbol (to set rela->sym later) */
	ALLOC_LINK(ksym_sec_sym, &kelf->symbols);
	ksym_sec_sym->sec = ksym_sec;
	ksym_sec_sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
	ksym_sec_sym->type = STT_SECTION;
	ksym_sec_sym->bind = STB_LOCAL;
	ksym_sec_sym->name = ".kpatch.symbols";

	/* lookup strings symbol */
	strsym = find_symbol_by_name(&kelf->symbols, ".kpatch.strings");
	if (!strsym)
		ERROR("can't find .kpatch.strings symbol");

	/* populate sections */
	index = 0;
	list_for_each_entry(relasec, &kelf->sections, list) {
		if (!is_rela_section(relasec))
			continue;
		if (!strcmp(relasec->name, ".rela.kpatch.funcs") ||
		    !strcmp(relasec->name, ".rela.kpatch.relocations") ||
		    !strcmp(relasec->name, ".rela.kpatch.symbols"))
			continue;

		special = false;
		for (s = special_sections; s->name; s++) {
			if ((s->arch & kelf->arch) == 0)
				continue;

			if (!strcmp(relasec->base->name, s->name))
				special = true;
		}

		list_for_each_entry_safe(rela, safe, &relasec->relas, list) {
			if (!rela->need_klp_reloc) {
				rela->sym->strip = SYMBOL_USED;
				continue;
			}

			/*
			 * Starting with Linux 5.8, .klp.arch sections are no
			 * longer supported: now that vmlinux relocations are
			 * written early, before paravirt and alternative
			 * module init, .klp.arch is technically not needed.
			 *
			 * For sanity we just need to make sure that there are
			 * no .klp.rela.{module}.{section} sections for special
			 * sections.  Otherwise there might be ordering issues,
			 * if the .klp.relas are applied after the module
			 * special section init code (e.g., apply_paravirt)
			 * runs due to late module patching.
			 */
			if (!KLP_ARCH && !vmlinux && special)
				ERROR("unsupported klp relocation reference to symbol '%s' in module-specific special section '%s'",
				      rela->sym->name, relasec->base->name);

			if (!lookup_symbol(table, rela->sym, &symbol))
				ERROR("can't find symbol '%s' in symbol table",
				      rela->sym->name);

			log_debug("lookup for %s: obj=%s sympos=%lu",
			          rela->sym->name, symbol.objname,
				  symbol.sympos);

			/* Fill in ksyms[index] */
			if (vmlinux)
				ksyms[index].src = symbol.addr;
			else
				/* for modules, src is discovered at runtime */
				ksyms[index].src = 0;
			ksyms[index].sympos = symbol.sympos;
			ksyms[index].type = rela->sym->type;
			ksyms[index].bind = rela->sym->bind;

			/* add rela to fill in ksyms[index].name field */
			ALLOC_LINK(rela2, &ksym_sec->rela->relas);
			rela2->sym = strsym;
			rela2->type = absolute_rela_type(kelf);
			rela2->addend = offset_of_string(&kelf->strings, rela->sym->name);
			rela2->offset = (unsigned int)(index * sizeof(*ksyms) + \
					offsetof(struct kpatch_symbol, name));

			/* add rela to fill in ksyms[index].objname field */
			ALLOC_LINK(rela2, &ksym_sec->rela->relas);
			rela2->sym = strsym;
			rela2->type = absolute_rela_type(kelf);
			rela2->addend = offset_of_string(&kelf->strings, symbol.objname);
			rela2->offset = (unsigned int)(index * sizeof(*ksyms) + \
					offsetof(struct kpatch_symbol, objname));

			/* Fill in krelas[index] */
			if (is_gcc6_localentry_bundled_sym(kelf, rela->sym) &&
			    rela->addend == (int)rela->sym->sym.st_value)
				rela->addend -= rela->sym->sym.st_value;
			krelas[index].addend = rela->addend;
			krelas[index].type = rela->type;
			krelas[index].external = !vmlinux && symbol.exported;

			/* add rela to fill in krelas[index].dest field */
			ALLOC_LINK(rela2, &krela_sec->rela->relas);
			if (!relasec->base->secsym) {
				struct symbol *sym;

				/*
				 * Newer toolchains are stingy with their
				 * section symbols, create one if it doesn't
				 * exist already.
				 */
				ALLOC_LINK(sym, &kelf->symbols);
				sym->sec = relasec->base;
				sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
				sym->type = STT_SECTION;
				sym->bind = STB_LOCAL;
				sym->name = relasec->base->name;
				relasec->base->secsym = sym;
			}
			rela2->sym = relasec->base->secsym;
			rela2->type = absolute_rela_type(kelf);
			rela2->addend = rela->offset;
			rela2->offset = (unsigned int)(index * sizeof(*krelas) + \
					offsetof(struct kpatch_relocation, dest));

			/* add rela to fill in krelas[index].objname field */
			ALLOC_LINK(rela2, &krela_sec->rela->relas);
			rela2->sym = strsym;
			rela2->type = absolute_rela_type(kelf);
			rela2->addend = offset_of_string(&kelf->strings, objname);
			rela2->offset = (unsigned int)(index * sizeof(*krelas) + \
				offsetof(struct kpatch_relocation, objname));

			/* add rela to fill in krelas[index].ksym field */
			ALLOC_LINK(rela2, &krela_sec->rela->relas);
			rela2->sym = ksym_sec_sym;
			rela2->type = absolute_rela_type(kelf);
			rela2->addend = (unsigned int)(index * sizeof(*ksyms));
			rela2->offset = (unsigned int)(index * sizeof(*krelas) + \
				offsetof(struct kpatch_relocation, ksym));

			/*
			 * Mark the referred to symbol for removal but
			 * only if it is not from this object file.
			 * The symbols from this object file may be needed
			 * later (for example, they may have relocations
			 * of their own which should be processed).
			 */
			if (!rela->sym->sec && rela->sym->strip != SYMBOL_USED)
				rela->sym->strip = SYMBOL_STRIP;
			list_del(&rela->list);
			free(rela);

			index++;
		}
	}

	/* set size to actual number of ksyms/krelas */
	ksym_sec->data->d_size = index * sizeof(struct kpatch_symbol);
	ksym_sec->sh.sh_size = ksym_sec->data->d_size;

	krela_sec->data->d_size = index * sizeof(struct kpatch_relocation);
	krela_sec->sh.sh_size = krela_sec->data->d_size;
}

static void kpatch_create_callbacks_objname_rela(struct kpatch_elf *kelf, char *objname)
{
	struct section *sec;
	struct rela *rela;
	struct symbol *strsym;
	int objname_offset;

	struct callback { char *name; int offset; };
	static struct callback callbacks[] = {
		{ .name = ".rela.kpatch.callbacks.pre_patch",
		  .offset = offsetof(struct kpatch_pre_patch_callback, objname) },
		{ .name = ".rela.kpatch.callbacks.post_patch",
		  .offset = offsetof(struct kpatch_post_patch_callback, objname) },
		{ .name = ".rela.kpatch.callbacks.pre_unpatch",
		  .offset = offsetof(struct kpatch_pre_unpatch_callback, objname) },
		{ .name = ".rela.kpatch.callbacks.post_unpatch",
		  .offset = offsetof(struct kpatch_post_patch_callback, objname) },
		{ .name = NULL, .offset = 0 },
	};
	struct callback *callbackp;

	/* lookup strings symbol */
	strsym = find_symbol_by_name(&kelf->symbols, ".kpatch.strings");
	if (!strsym)
		ERROR("can't find .kpatch.strings symbol");

	/* add objname to strings */
	objname_offset = offset_of_string(&kelf->strings, objname);

	list_for_each_entry(sec, &kelf->sections, list) {
		for (callbackp = callbacks; callbackp->name; callbackp++) {
			if (!strcmp(callbackp->name, sec->name)) {
				ALLOC_LINK(rela, &sec->relas);
				rela->sym = strsym;
				rela->type = absolute_rela_type(kelf);
				rela->addend = objname_offset;
				rela->offset = callbackp->offset;
				break;
			}
		}
	}
}

/*
 * Create links between text sections and their corresponding
 * __patchable_function_entries sections (as there may be multiple pfe
 * sections).
 */
static void kpatch_set_pfe_link(struct kpatch_elf *kelf)
{
	struct section* sec;
	struct rela *rela;

	if (!kelf->has_pfe)
		return;

	list_for_each_entry(sec, &kelf->sections, list) {
		if (strcmp(sec->name, "__patchable_function_entries"))
			continue;

		if (!sec->rela)
			continue;

		list_for_each_entry(rela, &sec->rela->relas, list)
			rela->sym->pfe = sec;
	}
}

/*
 * This function basically reimplements the functionality of the Linux
 * recordmcount script, so that patched functions can be recognized by ftrace.
 *
 * TODO: Eventually we can modify recordmount so that it recognizes our bundled
 * sections as valid and does this work for us.
 */
static void kpatch_create_ftrace_callsite_sections(struct kpatch_elf *kelf)
{
	int nr, index;
	struct section *sec = NULL;
	struct symbol *sym, *rela_sym;
	struct rela *rela;
	void **funcs;
	unsigned long insn_offset = 0;
	unsigned int rela_offset;

	nr = 0;
	list_for_each_entry(sym, &kelf->symbols, list)
		if (sym->type == STT_FUNC && sym->status != SAME &&
		    sym->has_func_profiling)
			nr++;

	if (!kelf->has_pfe)
		/*
		 * Create a single __mcount_loc section pair for all
		 * functions.
		 */
		sec = create_section_pair(kelf, "__mcount_loc", sizeof(void*), nr);

	/* populate sections */
	index = 0;
	list_for_each_entry(sym, &kelf->symbols, list) {
		if (sym->type != STT_FUNC || sym->status == SAME)
			continue;

		if (!sym->has_func_profiling) {
			log_debug("function %s has no ftrace callsite, no __patchable_function_entries/mcount record is needed\n",
				  sym->name);
			continue;
		}

		switch(kelf->arch) {
		case AARCH64: {
			unsigned char *insn = sym->sec->data->d_buf;
			int padding;
			int i;

			/*
			 * Skip the padding NOPs added by CALL_OPS.
			 */
			padding = function_padding_size(kelf, sym);
			insn += padding;

			/*
			 * If BTI (Branch Target Identification) is enabled then there
			 * might be an additional 'BTI C' instruction before the two
			 * patchable function entry 'NOP's.
			 * i.e. 0xd503245f (little endian)
			 */
			if (insn[0] == 0x5f) {
				if (insn[1] != 0x24 || insn[2] != 0x03 || insn[3] != 0xd5)
					ERROR("%s: unexpected instruction in patch section of function\n", sym->name);
				if (!padding)
					insn_offset += 4;
				insn += 4;
			}
			for (i = 0; i < 8; i += 4) {
				/* We expect a NOP i.e. 0xd503201f (little endian) */
				if (insn[i] != 0x1f || insn[i + 1] != 0x20 ||
				    insn[i + 2] != 0x03 || insn [i + 3] != 0xd5)
					ERROR("%s: unexpected instruction in patch section of function\n", sym->name);
			}
			break;
		}
		case PPC64: {
			unsigned char *insn;

			if (kelf->has_pfe) {
				insn_offset = sym->sym.st_value + PPC64_LOCAL_ENTRY_OFFSET(sym->sym.st_other);
				insn = sym->sec->data->d_buf + insn_offset;

				/* verify nop */
				if (insn[0] != 0x00 || insn[1] != 0x00 || insn[2] != 0x00 || insn[3] != 0x60)
					ERROR("%s: unexpected instruction in patch section of function\n", sym->name);
			} else {
				bool found = false;

				list_for_each_entry(rela, &sym->sec->rela->relas, list)
					if (!strcmp(rela->sym->name, "_mcount")) {
						found = true;
						break;
					}

				if (!found)
					ERROR("%s: unexpected missing call to _mcount()", __func__);

				insn_offset = rela->offset;
			}
			break;
		}
		case X86_64: {
			unsigned char *insn;
			void *newdata;

			rela = list_first_entry(&sym->sec->rela->relas, struct rela, list);

			/*
			 * For "call fentry", the relocation points to 1 byte past the
			 * beginning of the instruction.
			 */
			insn_offset = rela->offset - 1;

			/*
			 * R_X86_64_NONE is only generated by older versions of
			 * kernel/gcc which use the mcount script.  There's a
			 * NOP instead of a call to fentry.
			 */
			if (rela->type != R_X86_64_NONE)
				break;

			/* Make a writable copy of the text section data */
			newdata = malloc(sym->sec->data->d_size);
			if (!newdata)
				ERROR("malloc");
			memcpy(newdata, sym->sec->data->d_buf, sym->sec->data->d_size);
			sym->sec->data->d_buf = newdata;
			insn = newdata;

			/*
			 * Replace the NOP with a call to fentry.  The fentry
			 * rela symbol is already there, just need to change
			 * the relocation type accordingly.
			 */
			insn = sym->sec->data->d_buf;
			if (insn[0] != 0xf)
				ERROR("%s: unexpected instruction at the start of the function", sym->name);
			insn[0] = 0xe8;
			insn[1] = 0;
			insn[2] = 0;
			insn[3] = 0;
			insn[4] = 0;

			rela->type = R_X86_64_PC32;
			break;
		}
		case S390: {
			insn_offset = sym->sym.st_value;
			break;
		}
		default:
			ERROR("unsupported arch");
		}

		if (kelf->has_pfe) {
			/*
			 * Allocate a dedicated __patchable_function_entries for this function:
			 *   - its .sh_link will be updated by kpatch_reindex_elements()
			 *   - its lone rela is based on the section symbol
			 */
			sec = create_section_pair(kelf, "__patchable_function_entries", sizeof(void *), 1);
			sec->sh.sh_flags |= SHF_WRITE | SHF_ALLOC;
			/*
			 * Old linkers don't support mixing ordered and unordered sections, so set
			 * the SHF_LINK_ORDER flag only if it is set in the pfe section generated by
			 * the compiler.
			 */
			if (kelf->pfe_ordered)
				sec->sh.sh_flags |= SHF_LINK_ORDER;
			rela_sym = sym->sec->secsym;
			rela_offset = 0;
			rela_sym->pfe = sec;
		} else {
			/*
			 * mcount relas are based on the function symbol and saved in a
			 * single aggregate __mcount_loc section
			 */
			rela_sym = sym;
			rela_offset = (unsigned int) (index * sizeof(*funcs));
		}

		ALLOC_LINK(rela, &sec->rela->relas);
		rela->sym = rela_sym;
		rela->type = absolute_rela_type(kelf);
		rela->addend = insn_offset - rela->sym->sym.st_value;
		rela->offset = rela_offset;

		index++;
	}

	/* sanity check, index should equal nr */
	if (index != nr)
		ERROR("size mismatch in funcs sections");
}

/*
 * This function strips out symbols that were referenced by changed rela
 * sections, but the rela entries that referenced them were converted to
 * klp relocations and are no longer needed.
 */
static void kpatch_strip_unneeded_syms(struct kpatch_elf *kelf,
				       struct lookup_table *table)
{
	struct symbol *sym, *safe;

	list_for_each_entry_safe(sym, safe, &kelf->symbols, list) {
		if (sym->strip == SYMBOL_STRIP) {
			list_del(&sym->list);
			free(sym);
		}
	}
}

static void kpatch_create_strings_elements(struct kpatch_elf *kelf)
{
	struct section *sec;
	struct symbol *sym;

	/* create .kpatch.strings */

	/* allocate section resources */
	ALLOC_LINK(sec, &kelf->sections);
	sec->name = ".kpatch.strings";

	/* set data */
	sec->data = malloc(sizeof(*sec->data));
	if (!sec->data)
		ERROR("malloc");
	sec->data->d_type = ELF_T_BYTE;

	/* set section header */
	sec->sh.sh_type = SHT_PROGBITS;
	sec->sh.sh_entsize = 1;
	sec->sh.sh_addralign = 1;
	sec->sh.sh_flags = SHF_ALLOC;

	/* create .kpatch.strings section symbol (reuse sym variable) */

	ALLOC_LINK(sym, &kelf->symbols);
	sym->sec = sec;
	sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
	sym->type = STT_SECTION;
	sym->bind = STB_LOCAL;
	sym->name = ".kpatch.strings";
}

static void kpatch_build_strings_section_data(struct kpatch_elf *kelf)
{
	struct string *string;
	struct section *sec;
	size_t size;
	char *strtab;

	sec = find_section_by_name(&kelf->sections, ".kpatch.strings");
	if (!sec)
		ERROR("can't find .kpatch.strings");

	/* determine size */
	size = 0;
	list_for_each_entry(string, &kelf->strings, list)
		size += strlen(string->name) + 1;

	/* allocate section resources */
	strtab = malloc(size);
	if (!strtab)
		ERROR("malloc");
	sec->data->d_buf = strtab;
	sec->data->d_size = size;

	/* populate strings section data */
	list_for_each_entry(string, &kelf->strings, list) {
		strcpy(strtab, string->name);
		strtab += strlen(string->name) + 1;
	}
}

/*
 * Don't allow sibling calls from patched functions on ppc64le.  Before doing a
 * sibling call, the patched function restores the stack to its caller's stack.
 * The kernel-generated stub then writes the patch module's r2 (toc) value to
 * the caller's stack, corrupting it, eventually causing a panic after it
 * returns to the caller and the caller tries to use the livepatch module's toc
 * value.
 *
 * In theory we could instead a) generate a custom stub, or b) modify the
 * kernel livepatch_handler code to save/restore the stack r2 value, but this
 * is easier for now.
 */
static void kpatch_no_sibling_calls_ppc64le(struct kpatch_elf *kelf)
{
	struct symbol *sym;
	unsigned char *insn;
	unsigned int offset;
	int sibling_call_errors = 0;

	if (kelf->arch != PPC64)
		return;

	list_for_each_entry(sym, &kelf->symbols, list) {
		if (sym->type != STT_FUNC || sym->status != CHANGED)
			continue;

		for (offset = 0; offset < sym->sec->data->d_size; offset += 4) {

			insn = sym->sec->data->d_buf + offset;

			/*
			 * The instruction 0x48000000 can be assumed to be a
			 * sibling call:
			 *
			 * Bits 0-5 (opcode) == 0x9: unconditional branch
			 * Bit 30 (absolute) == 0: relative address
			 * Bit 31 (link) == 0: doesn't set LR (not a call)
			 *
			 * Bits 6-29 (branch address) == zero, which means
			 * it's either a branch to self (infinite loop), or
			 * there's a REL24 relocation for the address which
			 * will be written by the linker or the kernel.
			 */
			if (insn[3] != 0x48 || insn[2] != 0x00 ||
			    insn[1] != 0x00 || insn[0] != 0x00)
				continue;

			/* Make sure it's not a branch-to-self: */
			if (!find_rela_by_offset(sym->sec->rela, offset))
				continue;

			log_error("Found an unsupported sibling call at %s()+0x%lx.  Add __attribute__((optimize(\"-fno-optimize-sibling-calls\"))) to %s() definition.\n",
				  sym->name, sym->sym.st_value + offset, sym->name);
			sibling_call_errors++;
		}
	}

	if (sibling_call_errors)
		ERROR("Found %d unsupported sibling call(s) in the patched code.",
		      sibling_call_errors);
}

static bool kpatch_symbol_has_pfe_entry(struct kpatch_elf *kelf, struct symbol *sym)
{
	struct section *sec;
	struct rela *rela;

	if (!kelf->has_pfe)
		return false;

	list_for_each_entry(sec, &kelf->sections, list) {
		if (strcmp(sec->name, "__patchable_function_entries"))
			continue;
		if (!sec->rela)
			continue;

		list_for_each_entry(rela, &sec->rela->relas, list) {
			if (rela->sym->sec && sym->sec == rela->sym->sec &&
			    rela->sym->pfe == sec) {
				return true;
			}
		}
	}

	return false;
}

/* Check which functions have fentry/mcount calls; save this info for later use. */
static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf)
{
	struct symbol *sym;
	struct rela *rela;
	unsigned char *insn;
	list_for_each_entry(sym, &kelf->symbols, list) {
		if (sym->type != STT_FUNC || sym->is_pfx || !sym->sec)
			continue;

		switch(kelf->arch) {
		case PPC64:
			if (kpatch_symbol_has_pfe_entry(kelf, sym)) {
				sym->has_func_profiling = 1;
			} else if (sym->sec->rela) {
				list_for_each_entry(rela, &sym->sec->rela->relas, list) {
					if (!strcmp(rela->sym->name, "_mcount")) {
						sym->has_func_profiling = 1;
						break;
					}
				}
			}
			break;
		case X86_64:
			if (sym->sec->rela) {
				rela = list_first_entry(&sym->sec->rela->relas, struct rela,
							list);
				if ((rela->type != R_X86_64_NONE &&
				     rela->type != R_X86_64_PC32 &&
				     rela->type != R_X86_64_PLT32) ||
				    strcmp(rela->sym->name, "__fentry__"))
					continue;

				sym->has_func_profiling = 1;
			}
			break;
		case S390:
			/* Check for compiler generated fentry nop - jgnop 0 */
			insn = sym->sec->data->d_buf;
			if (insn[0] == 0xc0 && insn[1] == 0x04 &&
				insn[2] == 0x00 && insn[3] == 0x00 &&
				insn[4] == 0x00 && insn[5] == 0x00)
				sym->has_func_profiling = 1;
			break;
		case AARCH64:
			if (kpatch_symbol_has_pfe_entry(kelf, sym))
				sym->has_func_profiling = 1;
			break;
		default:
			ERROR("unsupported arch");
		}
	}
}

struct arguments {
	char *args[7];
	bool debug, klp_arch;
};

static char args_doc[] = "original.o patched.o parent-name parent-symtab Module.symvers patch-module-name output.o";

static struct argp_option options[] = {
	{"debug",	'd', NULL, 0, "Show debug output" },
	{"klp-arch",	'a', NULL, 0, "Kernel supports .klp.arch section" },
	{ NULL }
};

static error_t parse_opt (int key, char *arg, struct argp_state *state)
{
	/* Get the input argument from argp_parse, which we
	   know is a pointer to our arguments structure. */
	struct arguments *arguments = state->input;

	switch (key)
	{
		case 'd':
			arguments->debug = 1;
			break;
		case 'a':
			arguments->klp_arch = 1;
			break;
		case ARGP_KEY_ARG:
			if (state->arg_num >= 7)
				/* Too many arguments. */
				argp_usage (state);
			arguments->args[state->arg_num] = arg;
			break;
		case ARGP_KEY_END:
			if (state->arg_num < 7)
				/* Not enough arguments. */
				argp_usage (state);
			break;
		default:
			return ARGP_ERR_UNKNOWN;
	}
	return 0;
}

static struct argp argp = { options, parse_opt, args_doc, NULL };

int main(int argc, char *argv[])
{
	struct kpatch_elf *kelf_orig, *kelf_patched, *kelf_out;
	struct arguments arguments;
	int num_changed, callbacks_exist, new_globals_exist;
	struct lookup_table *lookup;
	struct section *relasec, *symtab;
	char *orig_obj, *patched_obj, *parent_name;
	char *parent_symtab, *mod_symvers, *patch_name, *output_obj;

	memset(&arguments, 0, sizeof(arguments));
	argp_parse (&argp, argc, argv, 0, NULL, &arguments);
	if (arguments.debug)
		loglevel = DEBUG;
	if (arguments.klp_arch)
		KLP_ARCH = true;

	elf_version(EV_CURRENT);

	orig_obj      = arguments.args[0];
	patched_obj   = arguments.args[1];
	parent_name   = arguments.args[2];
	parent_symtab = arguments.args[3];
	mod_symvers   = arguments.args[4];
	patch_name    = arguments.args[5];
	output_obj    = arguments.args[6];

	childobj = GET_CHILD_OBJ(orig_obj);

	kelf_orig = kpatch_elf_open(orig_obj);
	kelf_patched = kpatch_elf_open(patched_obj);

	kpatch_set_pfe_link(kelf_orig);
	kpatch_set_pfe_link(kelf_patched);

	kpatch_find_func_profiling_calls(kelf_orig);
	kpatch_find_func_profiling_calls(kelf_patched);

	kpatch_compare_elf_headers(kelf_orig->elf, kelf_patched->elf);
	kpatch_check_program_headers(kelf_orig->elf);
	kpatch_check_program_headers(kelf_patched->elf);

	kpatch_bundle_symbols(kelf_orig);
	kpatch_bundle_symbols(kelf_patched);

	kpatch_detect_child_functions(kelf_orig);
	kpatch_detect_child_functions(kelf_patched);

	lookup = lookup_open(parent_symtab, parent_name, mod_symvers, kelf_orig);

	kpatch_mark_grouped_sections(kelf_patched);
	kpatch_replace_sections_syms(kelf_orig);
	kpatch_replace_sections_syms(kelf_patched);

	kpatch_correlate_elfs(kelf_orig, kelf_patched);
	kpatch_correlate_static_local_variables(kelf_orig, kelf_patched);

	/*
	 * After this point, we don't care about kelf_orig anymore.
	 * We access its sections via the twin pointers in the
	 * section, symbol, and rela lists of kelf_patched.
	 */
	kpatch_mark_ignored_sections(kelf_patched);
	kpatch_compare_correlated_elements(kelf_patched);
	kpatch_mark_ignored_functions_same(kelf_patched);
	kpatch_mark_ignored_sections_same(kelf_patched);
	kpatch_check_func_profiling_calls(kelf_patched);
	kpatch_elf_teardown(kelf_orig);
	kpatch_elf_free(kelf_orig);

	kpatch_include_standard_elements(kelf_patched);
	num_changed = kpatch_include_changed_functions(kelf_patched);
	callbacks_exist = kpatch_include_callback_elements(kelf_patched);
	kpatch_include_force_elements(kelf_patched);
	new_globals_exist = kpatch_include_new_globals(kelf_patched);
	kpatch_include_debug_sections(kelf_patched);

	kpatch_process_special_sections(kelf_patched, lookup);

	kpatch_print_changes(kelf_patched);
	kpatch_dump_kelf(kelf_patched);

	kpatch_verify_patchability(kelf_patched);

	if (!num_changed && !new_globals_exist) {
		if (callbacks_exist)
			log_debug("no changed functions were found, but callbacks exist\n");
		else {
			log_debug("no changed functions were found\n");
			return EXIT_STATUS_NO_CHANGE;
		}
	}

	/* this is destructive to kelf_patched */
	kpatch_migrate_included_elements(kelf_patched, &kelf_out);
	kpatch_put_modinfo_namespaces(kelf_patched, kelf_out);

	/*
	 * Teardown kelf_patched since we shouldn't access sections or symbols
	 * through it anymore.  Don't free however, since our section and symbol
	 * name fields still point to strings in the Elf object owned by
	 * kpatch_patched.
	 */
	kpatch_elf_teardown(kelf_patched);

	kpatch_no_sibling_calls_ppc64le(kelf_out);

	/* create strings, patches, and klp relocation sections */
	kpatch_create_strings_elements(kelf_out);
	kpatch_create_patches_sections(kelf_out, lookup, parent_name);
	kpatch_create_intermediate_sections(kelf_out, lookup, parent_name, patch_name);
	kpatch_create_kpatch_arch_section(kelf_out, parent_name);
	kpatch_create_callbacks_objname_rela(kelf_out, parent_name);
	kpatch_build_strings_section_data(kelf_out);

	kpatch_create_ftrace_callsite_sections(kelf_out);

	/*
	 *  At this point, the set of output sections and symbols is
	 *  finalized.  Reorder the symbols into linker-compliant
	 *  order and index all the symbols and sections.  After the
	 *  indexes have been established, update index data
	 *  throughout the structure.
	 */
	kpatch_reorder_symbols(kelf_out);
	kpatch_strip_unneeded_syms(kelf_out, lookup);
	kpatch_reindex_elements(kelf_out);

	/*
	 * Update rela section headers and rebuild the rela section data
	 * buffers from the relas lists.
	 */
	symtab = find_section_by_name(&kelf_out->sections, ".symtab");
	if (!symtab)
		ERROR("missing .symtab section");

	list_for_each_entry(relasec, &kelf_out->sections, list) {
		if (!is_rela_section(relasec))
			continue;
		relasec->sh.sh_link = symtab->index;
		relasec->sh.sh_info = relasec->base->index;
		kpatch_rebuild_rela_section_data(relasec);
	}
	kpatch_check_relocations(kelf_out);

	kpatch_create_shstrtab(kelf_out);
	kpatch_create_strtab(kelf_out);
	kpatch_create_symtab(kelf_out);
	kpatch_dump_kelf(kelf_out);
	kpatch_write_output_elf(kelf_out, kelf_patched->elf, output_obj, 0664);

	lookup_close(lookup);
	kpatch_elf_free(kelf_patched);
	kpatch_elf_teardown(kelf_out);
	kpatch_elf_free(kelf_out);

	return EXIT_STATUS_SUCCESS;
}