I have been using xenomai all the time. I usually read some xenomai technical documents more or less and have a general understanding of xenomai. Recently, I also read some information about the Linux kernel of the operating system in order to find a job. I was watching Linux CNC in the afternoon, but I really didn't understand how he used the xenomai real-time kernel dynamically. Then I accidentally began to take a look at the source code of xenomai. It doesn't matter if you don't look at it. When you look at it, you suddenly feel enlightened. A lot of previously ignorant people suddenly become a little transparent. So I decided to go through the xenomai source code from today and explore the real-time implementation mechanism. I hope I can stick to it.
This article contains xenomai's personal learning opinions starting from 0, and will also throw out the relevant documents seen before as a small summary of graduate students' learning career.
The foundation is slightly weak. If you find any problems, you are welcome to correct them.

1, Things before xenomai started

The startup function of xenomai is

 xenomai_init(void)

The following functions are used in the source code to load into the kernel startup process.

device_initcall(xenomai_init);

as initcall mechanism of linux (for drivers compiled into the kernel) The situation described in. The xenomai module is loaded after the linux kernel is started. The loading process is as follows:

start_kernel  
	-> rest_init();
		-> kernel_thread(kernel_init, NULL, CLONE_FS);
			-> kernel_init()
				-> kernel_init_freeable();
					-> do_basic_setup();
						-> do_initcalls();  
							->do_initcall_level(0);
							````
							->do_initcall_level(6);

From then on, start the xenomai kernel;

2, xenomai_init(void) function

static int __init xenomai_init(void)
{
	int ret, __maybe_unused cpu;

	setup_init_state();

	if (!realtime_core_enabled()) {
		printk(XENO_WARNING "disabled on kernel command line\n");
		return 0;
	}

#ifdef CONFIG_SMP
	cpumask_clear(&xnsched_realtime_cpus);
	for_each_online_cpu(cpu) {
		if (supported_cpus_arg & (1UL << cpu))
			cpumask_set_cpu(cpu, &xnsched_realtime_cpus);
	}
	if (cpumask_empty(&xnsched_realtime_cpus)) {
		printk(XENO_WARNING "disabled via empty real-time CPU mask\n");
		set_realtime_core_state(COBALT_STATE_DISABLED);
		return 0;
	}
	cobalt_cpu_affinity = xnsched_realtime_cpus;
#endif /* CONFIG_SMP */

	xnsched_register_classes();

	ret = xnprocfs_init_tree();
	if (ret)
		goto fail;

	ret = mach_setup();
	if (ret)
		goto cleanup_proc;

	xnintr_mount();

	ret = xnpipe_mount();
	if (ret)
		goto cleanup_mach;

	ret = xnselect_mount();
	if (ret)
		goto cleanup_pipe;

	ret = sys_init();
	if (ret)
		goto cleanup_select;

	ret = mach_late_setup();
	if (ret)
		goto cleanup_sys;

	ret = rtdm_init();
	if (ret)
		goto cleanup_sys;

	ret = cobalt_init();
	if (ret)
		goto cleanup_rtdm;

	rtdm_fd_init();

	printk(XENO_INFO "Cobalt v%s %s%s%s%s\n",
	       XENO_VERSION_STRING,
	       boot_debug_notice,
	       boot_lat_trace_notice,
	       boot_evt_trace_notice,
	       boot_state_notice);

	return 0;

cleanup_rtdm:
	rtdm_cleanup();
cleanup_sys:
	sys_shutdown();
cleanup_select:
	xnselect_umount();
cleanup_pipe:
	xnpipe_umount();
cleanup_mach:
	mach_cleanup();
cleanup_proc:
	xnprocfs_cleanup_tree();
fail:
	set_realtime_core_state(COBALT_STATE_DISABLED);
	printk(XENO_ERR "init failed, code %d\n", ret);

	return ret;
}

It mainly includes the following functions

setup_init_state();
set_realtime_core_state();
xnsched_register_classes();
xnprocfs_init_tree();
mach_setup();
xnintr_mount();
xnpipe_mount();
xnselect_mount();
sys_init();
mach_late_setup();
rtdm_init();
cobalt_init();
rtdm_fd_init();

one by one

setup_init_state()

The source code is as follows

static void __init setup_init_state(void)
{
	static char warn_bad_state[] __initdata =
		XENO_WARNING "invalid init state '%s'\n";
	int n;

	for (n = 0; n < ARRAY_SIZE(init_states); n++)
		if (strcmp(init_states[n].label, init_state_arg) == 0) {
			set_realtime_core_state(init_states[n].state);
			return;
		}

	printk(warn_bad_state, init_state_arg);
}

Here are descriptions of other relevant parameters

static struct {
	const char *label;
	enum cobalt_run_states state;
} init_states[] __initdata = {
	{ "disabled", COBALT_STATE_DISABLED },
	{ "stopped", COBALT_STATE_STOPPED },
	{ "enabled", COBALT_STATE_WARMUP },
};
enum cobalt_run_states {
	COBALT_STATE_DISABLED,
	COBALT_STATE_RUNNING,
	COBALT_STATE_STOPPED,
	COBALT_STATE_TEARDOWN,
	COBALT_STATE_WARMUP,
};

xenomai maps its state description to enumerated value types

static char init_state_arg[16] = "enabled";

Enabled by default
Compare the status of the real-time core set when the corresponding character is transmitted

static inline void set_realtime_core_state(enum cobalt_run_states state)
{
	atomic_set(&cobalt_runstate, state);
}

Cobalt here_ Runstate is an atomic variable, which is actually an int

static inline void atomic_set(atomic_t *ptr, long v)
{
	ptr->v = v;
}
typedef struct { int v; } atomic_t;
atomic_t cobalt_runstate = ATOMIC_INIT(COBALT_STATE_WARMUP);
#define ATOMIC_INIT(__n) { (__n) }
//Cobalt here_ STATE_ Warmup is one of the previously defined xenomai states

The startup status can be set in / etc/default/grub

GRUB_CMDLINE_LINUX="isolcpus=0,1 xenomai.supported_cpus=0x03"

Official website Installing_Xenomai_3 Mentioned in
NAME
xenomai.state=
DESCRIPTION
Set the initial state of the Cobalt core at boot up, which may be enabled, stopped or disabled. See the documentation about the corectl(1) utility for a description of these states.
DEFAULT
enabled
corectl can perform input operations on the kernel state at the user level. The documents are as follows corectl - Cobalt core control interface
The tool is installed in / usr/xenomai/sbin by default
After setting the xenomai status, you will check the first hand. If it is not set successfully or there are other problems. Exit as follows

	if (!realtime_core_enabled()) {
		printk(XENO_WARNING "disabled on kernel command line\n");
		return 0;
	}

Episode smp

I mentioned it in my previous article Install ubuntu +xenomai3.1 patch for virtual machine
Now computers are basically multi - core, and this option is turned on by default

#ifdef CONFIG_SMP
	cpumask_clear(&xnsched_realtime_cpus);
	for_each_online_cpu(cpu) {
		if (supported_cpus_arg & (1UL << cpu))
			cpumask_set_cpu(cpu, &xnsched_realtime_cpus);
	}
	if (cpumask_empty(&xnsched_realtime_cpus)) {
		printk(XENO_WARNING "disabled via empty real-time CPU mask\n");
		set_realtime_core_state(COBALT_STATE_DISABLED);
		return 0;
	}
	cobalt_cpu_affinity = xnsched_realtime_cpus;
#endif /* CONFIG_SMP */

cpumask_clear as the name implies, the mask is cleared, which is defined as follows

/**
 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
 * @dstp: the cpumask pointer
 */
static inline void cpumask_clear(struct cpumask *dstp)
{
	bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
}
#define cpumask_bits(maskp) ((maskp)->bits)
static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
{
	if (small_const_nbits(nbits))
		*dst = 0UL;
	else {
		unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
		memset(dst, 0, len);
	}
}

Then set the cpu mask supported by xenomai

#define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)
#define for_each_cpu(cpu, mask)			\
	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)

I don't know why the cpu only traverses 0 and then sets the cpu mask
You can also set grub parameters here. See the official website [installing] for details_ Xenomai_ 3].

cobalt_cpu_affinity = xnsched_realtime_cpus;

Then the cpu affinity of xenomai will be bound to the set cpu. linux will not use these two cores.
There may still be problems here, which will be added later

xnsched_register_classes()

void xnsched_register_classes(void)
{
	xnsched_register_class(&xnsched_class_idle);
#ifdef CONFIG_XENO_OPT_SCHED_WEAK
	xnsched_register_class(&xnsched_class_weak);
#endif
#ifdef CONFIG_XENO_OPT_SCHED_TP
	xnsched_register_class(&xnsched_class_tp);
#endif
#ifdef CONFIG_XENO_OPT_SCHED_SPORADIC
	xnsched_register_class(&xnsched_class_sporadic);
#endif
#ifdef CONFIG_XENO_OPT_SCHED_QUOTA
	xnsched_register_class(&xnsched_class_quota);
#endif
	xnsched_register_class(&xnsched_class_rt);
}

dmesg shows that xenomai only registers the following two

[    1.629946] [Xenomai] scheduling class idle registered.
[    1.629946] [Xenomai] scheduling class rt registered.

Xenomai source code analysis Chapter 1 - xenomai_init