/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Benchmarking code execution time inside the kernel
*
* Copyright (C) 2014, Red Hat, Inc., Jesper Dangaard Brouer
* for licensing details see kernel-base/COPYING
*/
#ifndef _LINUX_TIME_BENCH_H
#define _LINUX_TIME_BENCH_H
/* Main structure used for recording a benchmark run */
struct time_bench_record {
uint32_t version_abi;
uint32_t loops;
/* Requested loop invocations */
uint32_t step;
/* option for e.g. bulk invocations */
uint32_t flags;
/* Measurements types enabled */
#define TIME_BENCH_LOOP BIT(0)
#define TIME_BENCH_TSC BIT(1)
#define TIME_BENCH_WALLCLOCK BIT(2)
#define TIME_BENCH_PMU BIT(3)
uint32_t cpu;
/* Used when embedded in time_bench_cpu */
/* Records */
uint64_t invoked_cnt;
/* Returned actual invocations */
uint64_t tsc_start;
uint64_t tsc_stop;
struct timespec64 ts_start;
struct timespec64 ts_stop;
/* PMU counters for instruction and cycles
* instructions counter including pipelined instructions
*/
uint64_t pmc_inst_start;
uint64_t pmc_inst_stop;
/* CPU unhalted clock counter */
uint64_t pmc_clk_start;
uint64_t pmc_clk_stop;
/* Result records */
uint64_t tsc_interval;
uint64_t time_start, time_stop, time_interval;
/* in nanosec */
uint64_t pmc_inst, pmc_clk;
/* Derived result records */
uint64_t tsc_cycles;
// +decimal?
uint64_t ns_per_call_quotient, ns_per_call_decimal;
uint64_t time_sec;
uint32_t time_sec_remainder;
uint64_t pmc_ipc_quotient, pmc_ipc_decimal;
/* inst per cycle */
};
/* For synchronizing parallel CPUs to run concurrently */
struct time_bench_sync {
atomic_t nr_tests_running;
struct completion start_event;
};
/* Keep track of CPUs executing our bench function.
*
* Embed a time_bench_record for storing info per cpu
*/
struct time_bench_cpu {
struct time_bench_record rec;
struct time_bench_sync *sync;
/* back ptr */
struct task_struct *task;
/* "data" opaque could have been placed in time_bench_sync,
* but to avoid any false sharing, place it per CPU
*/
void *data;
/* Support masking outsome CPUs, mark if it ran */
bool did_bench_run;
/* int cpu; // note CPU stored in time_bench_record */
int (*bench_func)(
struct time_bench_record *record,
void *data);
};
/*
* Below TSC assembler code is not compatible with other archs, and
* can also fail on guests if cpu-flags are not correct.
*
* The way TSC reading is used, many iterations, does not require as
* high accuracy as described below (in Intel Doc #324264).
*
* Considering changing to use get_cycles() (#include <asm/timex.h>).
*/
/** TSC (Time-Stamp Counter) based **
* Recommend reading, to understand details of reading TSC accurately:
* Intel Doc #324264, "How to Benchmark Code Execution Times on Intel"
*
* Consider getting exclusive ownership of CPU by using:
* unsigned long flags;
* preempt_disable();
* raw_local_irq_save(flags);
* _your_code_
* raw_local_irq_restore(flags);
* preempt_enable();
*
* Clobbered registers: "%rax", "%rbx", "%rcx", "%rdx"
* RDTSC only change "%rax" and "%rdx" but
* CPUID clears the high 32-bits of all (rax/rbx/rcx/rdx)
*/
static __always_inline uint64_t tsc_start_clock(
void)
{
/* See: Intel Doc #324264 */
unsigned int hi, lo;
asm volatile(
"CPUID\n\t"
"RDTSC\n\t"
"mov %%edx, %0\n\t"
"mov %%eax, %1\n\t"
:
"=r"(hi),
"=r"(lo)::
"%rax",
"%rbx",
"%rcx",
"%rdx");
//FIXME: on 32bit use clobbered %eax + %edx
return ((uint64_t)lo) | (((uint64_t)hi) << 32);
}
static __always_inline uint64_t tsc_stop_clock(
void)
{
/* See: Intel Doc #324264 */
unsigned int hi, lo;
asm volatile(
"RDTSCP\n\t"
"mov %%edx, %0\n\t"
"mov %%eax, %1\n\t"
"CPUID\n\t"
:
"=r"(hi),
"=r"(lo)::
"%rax",
"%rbx",
"%rcx",
"%rdx");
return ((uint64_t)lo) | (((uint64_t)hi) << 32);
}
/** Wall-clock based **
*
* use: getnstimeofday()
* getnstimeofday(&rec->ts_start);
* getnstimeofday(&rec->ts_stop);
*
* API changed see: Documentation/core-api/timekeeping.rst
* https://www.kernel.org/doc/html/latest/core-api/timekeeping.html#c.getnstimeofday
*
* We should instead use: ktime_get_real_ts64() is a direct
* replacement, but consider using monotonic time (ktime_get_ts64())
* and/or a ktime_t based interface (ktime_get()/ktime_get_real()).
*/
/** PMU (Performance Monitor Unit) based **
*
* Needed for calculating: Instructions Per Cycle (IPC)
* - The IPC number tell how efficient the CPU pipelining were
*/
//lookup: perf_event_create_kernel_counter()
bool time_bench_PMU_config(
bool enable);
/* Raw reading via rdpmc() using fixed counters
*
* From: https://github.com/andikleen/simple-pmu
*/
enum {
FIXED_SELECT = (1U << 30),
/* == 0x40000000 */
FIXED_INST_RETIRED_ANY = 0,
FIXED_CPU_CLK_UNHALTED_CORE = 1,
FIXED_CPU_CLK_UNHALTED_REF = 2,
};
static __always_inline
unsigned int long long p_rdpmc(
unsigned int in)
{
unsigned int d, a;
asm volatile(
"rdpmc" :
"=d"(d),
"=a"(a) :
"c"(in) :
"memory");
return ((
unsigned long long)d << 32) | a;
}
/* These PMU counter needs to be enabled, but I don't have the
* configure code implemented. My current hack is running:
* sudo perf stat -e cycles:k -e instructions:k insmod lib/ring_queue_test.ko
*/
/* Reading all pipelined instruction */
static __always_inline
unsigned long long pmc_inst(
void)
{
return p_rdpmc(FIXED_SELECT | FIXED_INST_RETIRED_ANY);
}
/* Reading CPU clock cycles */
static __always_inline
unsigned long long pmc_clk(
void)
{
return p_rdpmc(FIXED_SELECT | FIXED_CPU_CLK_UNHALTED_CORE);
}
/* Raw reading via MSR rdmsr() is likely wrong
* FIXME: How can I know which raw MSR registers are conf for what?
*/
#define MSR_IA32_PCM0 0x400000C1
/* PERFCTR0 */
#define MSR_IA32_PCM1 0x400000C2
/* PERFCTR1 */
#define MSR_IA32_PCM2 0x400000C3
static inline uint64_t msr_inst(
unsigned long long *msr_result)
{
return rdmsrq_safe(MSR_IA32_PCM0, msr_result);
}
/** Generic functions **
*/
bool time_bench_loop(uint32_t loops,
int step,
char *txt,
void *data,
int (*func)(
struct time_bench_record *rec,
void *data));
bool time_bench_calc_stats(
struct time_bench_record *rec);
void time_bench_run_concurrent(uint32_t loops,
int step,
void *data,
const struct cpumask *mask,
/* Support masking outsome CPUs*/
struct time_bench_sync *sync,
struct time_bench_cpu *cpu_tasks,
int (*func)(
struct time_bench_record *record,
void *data));
void time_bench_print_stats_cpumask(
const char *desc,
struct time_bench_cpu *cpu_tasks,
const struct cpumask *mask);
//FIXME: use rec->flags to select measurement, should be MACRO
static __always_inline
void time_bench_start(
struct time_bench_record *rec)
{
//getnstimeofday(&rec->ts_start);
ktime_get_real_ts64(&rec->ts_start);
if (rec->flags & TIME_BENCH_PMU) {
rec->pmc_inst_start = pmc_inst();
rec->pmc_clk_start = pmc_clk();
}
rec->tsc_start = tsc_start_clock();
}
static __always_inline
void time_bench_stop(
struct time_bench_record *rec,
uint64_t invoked_cnt)
{
rec->tsc_stop = tsc_stop_clock();
if (rec->flags & TIME_BENCH_PMU) {
rec->pmc_inst_stop = pmc_inst();
rec->pmc_clk_stop = pmc_clk();
}
//getnstimeofday(&rec->ts_stop);
ktime_get_real_ts64(&rec->ts_stop);
rec->invoked_cnt = invoked_cnt;
}
#endif /* _LINUX_TIME_BENCH_H */
