p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'fortglx/4.6/time' of https://git.linaro.org/people/john.stultz/linux into timers/core 

Pull the cross-timestamp infrastructure from John Stultz.

Allows precise correlation of device timestamps with system time. Primary use
cases being PTP and audio.
This commit is contained in:
Thomas Gleixner 2016-03-04 08:12:27 +01:00
parent 82e88ff1ea 01d7ada57e
commit 6936527233
15 changed files with 548 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Documentation/ptp/testptp.c
View File

@ -277,13 +277,15 @@ int main(int argc, char *argv[])
" %d external time stamp channels\n"
" %d programmable periodic signals\n"
" %d pulse per second\n"
" %d programmable pins\n",
" %d programmable pins\n"
" %d cross timestamping\n",
caps.max_adj,
caps.n_alarm,
caps.n_ext_ts,
caps.n_per_out,
caps.pps,
caps.n_pins);
caps.n_pins,
caps.cross_timestamping);
}
}

2
arch/x86/include/asm/cpufeature.h
View File

@ -85,7 +85,7 @@
#define X86_FEATURE_P4 ( 3*32+ 7) /* "" P4 */
#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */
#define X86_FEATURE_UP ( 3*32+ 9) /* smp kernel running on up */
/* free, was #define X86_FEATURE_FXSAVE_LEAK ( 3*32+10) * "" FXSAVE leaks FOP/FIP/FOP */
#define X86_FEATURE_ART (3*32+10) /* Platform has always running timer (ART) */
#define X86_FEATURE_ARCH_PERFMON ( 3*32+11) /* Intel Architectural PerfMon */
#define X86_FEATURE_PEBS ( 3*32+12) /* Precise-Event Based Sampling */
#define X86_FEATURE_BTS ( 3*32+13) /* Branch Trace Store */

2
arch/x86/include/asm/tsc.h
View File

@ -29,6 +29,8 @@ static inline cycles_t get_cycles(void)
return rdtsc();
}
extern struct system_counterval_t convert_art_to_tsc(cycle_t art);
extern void tsc_init(void);
extern void mark_tsc_unstable(char *reason);
extern int unsynchronized_tsc(void);

59
arch/x86/kernel/tsc.c
View File

@ -43,6 +43,11 @@ static DEFINE_STATIC_KEY_FALSE(__use_tsc);
int tsc_clocksource_reliable;
static u32 art_to_tsc_numerator;
static u32 art_to_tsc_denominator;
static u64 art_to_tsc_offset;
struct clocksource *art_related_clocksource;
/*
* Use a ring-buffer like data structure, where a writer advances the head by
* writing a new data entry and a reader advances the tail when it observes a
@ -964,6 +969,37 @@ core_initcall(cpufreq_tsc);
#endif /* CONFIG_CPU_FREQ */
#define ART_CPUID_LEAF (0x15)
#define ART_MIN_DENOMINATOR (1)
/*
* If ART is present detect the numerator:denominator to convert to TSC
*/
static void detect_art(void)
{
unsigned int unused[2];
if (boot_cpu_data.cpuid_level < ART_CPUID_LEAF)
return;
cpuid(ART_CPUID_LEAF, &art_to_tsc_denominator,
&art_to_tsc_numerator, unused, unused+1);
/* Don't enable ART in a VM, non-stop TSC required */
if (boot_cpu_has(X86_FEATURE_HYPERVISOR) ||
!boot_cpu_has(X86_FEATURE_NONSTOP_TSC) ||
art_to_tsc_denominator < ART_MIN_DENOMINATOR)
return;
if (rdmsrl_safe(MSR_IA32_TSC_ADJUST, &art_to_tsc_offset))
return;
/* Make this sticky over multiple CPU init calls */
setup_force_cpu_cap(X86_FEATURE_ART);
}
/* clocksource code */
static struct clocksource clocksource_tsc;
@ -1071,6 +1107,25 @@ int unsynchronized_tsc(void)
return 0;
}
/*
* Convert ART to TSC given numerator/denominator found in detect_art()
*/
struct system_counterval_t convert_art_to_tsc(cycle_t art)
{
u64 tmp, res, rem;
rem = do_div(art, art_to_tsc_denominator);
res = art * art_to_tsc_numerator;
tmp = rem * art_to_tsc_numerator;
do_div(tmp, art_to_tsc_denominator);
res += tmp + art_to_tsc_offset;
return (struct system_counterval_t) {.cs = art_related_clocksource,
.cycles = res};
}
EXPORT_SYMBOL(convert_art_to_tsc);
static void tsc_refine_calibration_work(struct work_struct *work);
static DECLARE_DELAYED_WORK(tsc_irqwork, tsc_refine_calibration_work);
@ -1142,6 +1197,8 @@ static void tsc_refine_calibration_work(struct work_struct *work)
(unsigned long)tsc_khz % 1000);
out:
if (boot_cpu_has(X86_FEATURE_ART))
art_related_clocksource = &clocksource_tsc;
clocksource_register_khz(&clocksource_tsc, tsc_khz);
}
@ -1235,6 +1292,8 @@ void __init tsc_init(void)
mark_tsc_unstable("TSCs unsynchronized");
check_system_tsc_reliable();
detect_art();
}
#ifdef CONFIG_SMP

9
drivers/net/ethernet/intel/Kconfig
View File

@ -83,6 +83,15 @@ config E1000E
To compile this driver as a module, choose M here. The module
will be called e1000e.
config E1000E_HWTS
bool "Support HW cross-timestamp on PCH devices"
default y
depends on E1000E && X86
---help---
Say Y to enable hardware supported cross-timestamping on PCH
devices. The cross-timestamp is available through the PTP clock
driver precise cross-timestamp ioctl (PTP_SYS_OFFSET_PRECISE).
config IGB
tristate "Intel(R) 82575/82576 PCI-Express Gigabit Ethernet support"
depends on PCI

5
drivers/net/ethernet/intel/e1000e/defines.h
View File

@ -528,6 +528,11 @@
#define E1000_RXCW_C 0x20000000 /* Receive config */
#define E1000_RXCW_SYNCH 0x40000000 /* Receive config synch */
/* HH Time Sync */
#define E1000_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK 0x0000F000 /* max delay */
#define E1000_TSYNCTXCTL_SYNC_COMP 0x40000000 /* sync complete */
#define E1000_TSYNCTXCTL_START_SYNC 0x80000000 /* initiate sync */
#define E1000_TSYNCTXCTL_VALID 0x00000001 /* Tx timestamp valid */
#define E1000_TSYNCTXCTL_ENABLED 0x00000010 /* enable Tx timestamping */

85
drivers/net/ethernet/intel/e1000e/ptp.c
View File

@ -26,6 +26,12 @@
#include "e1000.h"
#ifdef CONFIG_E1000E_HWTS
#include <linux/clocksource.h>
#include <linux/ktime.h>
#include <asm/tsc.h>
#endif
/**
* e1000e_phc_adjfreq - adjust the frequency of the hardware clock
* @ptp: ptp clock structure
@ -98,6 +104,78 @@ static int e1000e_phc_adjtime(struct ptp_clock_info *ptp, s64 delta)
return 0;
}
#ifdef CONFIG_E1000E_HWTS
#define MAX_HW_WAIT_COUNT (3)
/**
* e1000e_phc_get_syncdevicetime - Callback given to timekeeping code reads system/device registers
* @device: current device time
* @system: system counter value read synchronously with device time
* @ctx: context provided by timekeeping code
*
* Read device and system (ART) clock simultaneously and return the corrected
* clock values in ns.
**/
static int e1000e_phc_get_syncdevicetime(ktime_t *device,
struct system_counterval_t *system,
void *ctx)
{
struct e1000_adapter *adapter = (struct e1000_adapter *)ctx;
struct e1000_hw *hw = &adapter->hw;
unsigned long flags;
int i;
u32 tsync_ctrl;
cycle_t dev_cycles;
cycle_t sys_cycles;
tsync_ctrl = er32(TSYNCTXCTL);
tsync_ctrl |= E1000_TSYNCTXCTL_START_SYNC |
E1000_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK;
ew32(TSYNCTXCTL, tsync_ctrl);
for (i = 0; i < MAX_HW_WAIT_COUNT; ++i) {
udelay(1);
tsync_ctrl = er32(TSYNCTXCTL);
if (tsync_ctrl & E1000_TSYNCTXCTL_SYNC_COMP)
break;
}
if (i == MAX_HW_WAIT_COUNT)
return -ETIMEDOUT;
dev_cycles = er32(SYSSTMPH);
dev_cycles <<= 32;
dev_cycles |= er32(SYSSTMPL);
spin_lock_irqsave(&adapter->systim_lock, flags);
*device = ns_to_ktime(timecounter_cyc2time(&adapter->tc, dev_cycles));
spin_unlock_irqrestore(&adapter->systim_lock, flags);
sys_cycles = er32(PLTSTMPH);
sys_cycles <<= 32;
sys_cycles |= er32(PLTSTMPL);
*system = convert_art_to_tsc(sys_cycles);
return 0;
}
/**
* e1000e_phc_getsynctime - Reads the current system/device cross timestamp
* @ptp: ptp clock structure
* @cts: structure containing timestamp
*
* Read device and system (ART) clock simultaneously and return the scaled
* clock values in ns.
**/
static int e1000e_phc_getcrosststamp(struct ptp_clock_info *ptp,
struct system_device_crosststamp *xtstamp)
{
struct e1000_adapter *adapter = container_of(ptp, struct e1000_adapter,
ptp_clock_info);
return get_device_system_crosststamp(e1000e_phc_get_syncdevicetime,
adapter, NULL, xtstamp);
}
#endif/*CONFIG_E1000E_HWTS*/
/**
* e1000e_phc_gettime - Reads the current time from the hardware clock
* @ptp: ptp clock structure
@ -236,6 +314,13 @@ void e1000e_ptp_init(struct e1000_adapter *adapter)
break;
}
#ifdef CONFIG_E1000E_HWTS
/* CPU must have ART and GBe must be from Sunrise Point or greater */
if (hw->mac.type >= e1000_pch_spt && boot_cpu_has(X86_FEATURE_ART))
adapter->ptp_clock_info.getcrosststamp =
e1000e_phc_getcrosststamp;
#endif/*CONFIG_E1000E_HWTS*/
INIT_DELAYED_WORK(&adapter->systim_overflow_work,
e1000e_systim_overflow_work);

4
drivers/net/ethernet/intel/e1000e/regs.h
View File

@ -245,6 +245,10 @@
#define E1000_SYSTIML 0x0B600 /* System time register Low - RO */
#define E1000_SYSTIMH 0x0B604 /* System time register High - RO */
#define E1000_TIMINCA 0x0B608 /* Increment attributes register - RW */
#define E1000_SYSSTMPL 0x0B648 /* HH Timesync system stamp low register */
#define E1000_SYSSTMPH 0x0B64C /* HH Timesync system stamp hi register */
#define E1000_PLTSTMPL 0x0B640 /* HH Timesync platform stamp low register */
#define E1000_PLTSTMPH 0x0B644 /* HH Timesync platform stamp hi register */
#define E1000_RXMTRL 0x0B634 /* Time sync Rx EtherType and Msg Type - RW */
#define E1000_RXUDP 0x0B638 /* Time Sync Rx UDP Port - RW */

27
drivers/ptp/ptp_chardev.c
View File

@ -22,6 +22,7 @@
#include <linux/poll.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/timekeeping.h>
#include "ptp_private.h"
@ -120,11 +121,13 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
struct ptp_clock_caps caps;
struct ptp_clock_request req;
struct ptp_sys_offset *sysoff = NULL;
struct ptp_sys_offset_precise precise_offset;
struct ptp_pin_desc pd;
struct ptp_clock *ptp = container_of(pc, struct ptp_clock, clock);
struct ptp_clock_info *ops = ptp->info;
struct ptp_clock_time *pct;
struct timespec64 ts;
struct system_device_crosststamp xtstamp;
int enable, err = 0;
unsigned int i, pin_index;
@ -138,6 +141,7 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
caps.n_per_out = ptp->info->n_per_out;
caps.pps = ptp->info->pps;
caps.n_pins = ptp->info->n_pins;
caps.cross_timestamping = ptp->info->getcrosststamp != NULL;
if (copy_to_user((void __user *)arg, &caps, sizeof(caps)))
err = -EFAULT;
break;
@ -180,6 +184,29 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
err = ops->enable(ops, &req, enable);
break;
case PTP_SYS_OFFSET_PRECISE:
if (!ptp->info->getcrosststamp) {
err = -EOPNOTSUPP;
break;
}
err = ptp->info->getcrosststamp(ptp->info, &xtstamp);
if (err)
break;
ts = ktime_to_timespec64(xtstamp.device);
precise_offset.device.sec = ts.tv_sec;
precise_offset.device.nsec = ts.tv_nsec;
ts = ktime_to_timespec64(xtstamp.sys_realtime);
precise_offset.sys_realtime.sec = ts.tv_sec;
precise_offset.sys_realtime.nsec = ts.tv_nsec;
ts = ktime_to_timespec64(xtstamp.sys_monoraw);
precise_offset.sys_monoraw.sec = ts.tv_sec;
precise_offset.sys_monoraw.nsec = ts.tv_nsec;
if (copy_to_user((void __user *)arg, &precise_offset,
sizeof(precise_offset)))
err = -EFAULT;
break;
case PTP_SYS_OFFSET:
sysoff = kmalloc(sizeof(*sysoff), GFP_KERNEL);
if (!sysoff) {

21
include/linux/pps_kernel.h
View File

@ -111,22 +111,17 @@ static inline void timespec_to_pps_ktime(struct pps_ktime *kt,
kt->nsec = ts.tv_nsec;
}
static inline void pps_get_ts(struct pps_event_time *ts)
{
struct system_time_snapshot snap;
ktime_get_snapshot(&snap);
ts->ts_real = ktime_to_timespec64(snap.real);
#ifdef CONFIG_NTP_PPS
static inline void pps_get_ts(struct pps_event_time *ts)
{
ktime_get_raw_and_real_ts64(&ts->ts_raw, &ts->ts_real);
ts->ts_raw = ktime_to_timespec64(snap.raw);
#endif
}
#else /* CONFIG_NTP_PPS */
static inline void pps_get_ts(struct pps_event_time *ts)
{
ktime_get_real_ts64(&ts->ts_real);
}
#endif /* CONFIG_NTP_PPS */
/* Subtract known time delay from PPS event time(s) */
static inline void pps_sub_ts(struct pps_event_time *ts, struct timespec64 delta)
{

8
include/linux/ptp_clock_kernel.h
View File

@ -38,6 +38,7 @@ struct ptp_clock_request {
};
};
struct system_device_crosststamp;
/**
* struct ptp_clock_info - decribes a PTP hardware clock
*
@ -67,6 +68,11 @@ struct ptp_clock_request {
* @gettime64: Reads the current time from the hardware clock.
* parameter ts: Holds the result.
*
* @getcrosststamp: Reads the current time from the hardware clock and
* system clock simultaneously.
* parameter cts: Contains timestamp (device,system) pair,
* where system time is realtime and monotonic.
*
* @settime64: Set the current time on the hardware clock.
* parameter ts: Time value to set.
*
@ -105,6 +111,8 @@ struct ptp_clock_info {
int (*adjfreq)(struct ptp_clock_info *ptp, s32 delta);
int (*adjtime)(struct ptp_clock_info *ptp, s64 delta);
int (*gettime64)(struct ptp_clock_info *ptp, struct timespec64 *ts);
int (*getcrosststamp)(struct ptp_clock_info *ptp,
struct system_device_crosststamp *cts);
int (*settime64)(struct ptp_clock_info *p, const struct timespec64 *ts);
int (*enable)(struct ptp_clock_info *ptp,
struct ptp_clock_request *request, int on);

2
include/linux/timekeeper_internal.h
View File

@ -50,6 +50,7 @@ struct tk_read_base {
* @offs_tai: Offset clock monotonic -> clock tai
* @tai_offset: The current UTC to TAI offset in seconds
* @clock_was_set_seq: The sequence number of clock was set events
* @cs_was_changed_seq: The sequence number of clocksource change events
* @next_leap_ktime: CLOCK_MONOTONIC time value of a pending leap-second
* @raw_time: Monotonic raw base time in timespec64 format
* @cycle_interval: Number of clock cycles in one NTP interval
@ -91,6 +92,7 @@ struct timekeeper {
ktime_t offs_tai;
s32 tai_offset;
unsigned int clock_was_set_seq;
u8 cs_was_changed_seq;
ktime_t next_leap_ktime;
struct timespec64 raw_time;

58
include/linux/timekeeping.h
View File

@ -266,6 +266,64 @@ extern void timekeeping_inject_sleeptime64(struct timespec64 *delta);
extern void ktime_get_raw_and_real_ts64(struct timespec64 *ts_raw,
struct timespec64 *ts_real);
/*
* struct system_time_snapshot - simultaneous raw/real time capture with
* counter value
* @cycles: Clocksource counter value to produce the system times
* @real: Realtime system time
* @raw: Monotonic raw system time
* @clock_was_set_seq: The sequence number of clock was set events
* @cs_was_changed_seq: The sequence number of clocksource change events
*/
struct system_time_snapshot {
cycle_t cycles;
ktime_t real;
ktime_t raw;
unsigned int clock_was_set_seq;
u8 cs_was_changed_seq;
};
/*
* struct system_device_crosststamp - system/device cross-timestamp
* (syncronized capture)
* @device: Device time
* @sys_realtime: Realtime simultaneous with device time
* @sys_monoraw: Monotonic raw simultaneous with device time
*/
struct system_device_crosststamp {
ktime_t device;
ktime_t sys_realtime;
ktime_t sys_monoraw;
};
/*
* struct system_counterval_t - system counter value with the pointer to the
* corresponding clocksource
* @cycles: System counter value
* @cs: Clocksource corresponding to system counter value. Used by
* timekeeping code to verify comparibility of two cycle values
*/
struct system_counterval_t {
cycle_t cycles;
struct clocksource *cs;
};
/*
* Get cross timestamp between system clock and device clock
*/
extern int get_device_system_crosststamp(
int (*get_time_fn)(ktime_t *device_time,
struct system_counterval_t *system_counterval,
void *ctx),
void *ctx,
struct system_time_snapshot *history,
struct system_device_crosststamp *xtstamp);
/*
* Simultaneously snapshot realtime and monotonic raw clocks
*/
extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot);
/*
* Persistent clock related interfaces
*/

13
include/uapi/linux/ptp_clock.h
View File

@ -51,7 +51,9 @@ struct ptp_clock_caps {
int n_per_out; /* Number of programmable periodic signals. */
int pps; /* Whether the clock supports a PPS callback. */
int n_pins; /* Number of input/output pins. */
int rsv[14]; /* Reserved for future use. */
/* Whether the clock supports precise system-device cross timestamps */
int cross_timestamping;
int rsv[13]; /* Reserved for future use. */
};
struct ptp_extts_request {
@ -81,6 +83,13 @@ struct ptp_sys_offset {
struct ptp_clock_time ts[2 * PTP_MAX_SAMPLES + 1];
};
struct ptp_sys_offset_precise {
struct ptp_clock_time device;
struct ptp_clock_time sys_realtime;
struct ptp_clock_time sys_monoraw;
unsigned int rsv[4]; /* Reserved for future use. */
};
enum ptp_pin_function {
PTP_PF_NONE,
PTP_PF_EXTTS,
@ -124,6 +133,8 @@ struct ptp_pin_desc {
#define PTP_SYS_OFFSET _IOW(PTP_CLK_MAGIC, 5, struct ptp_sys_offset)
#define PTP_PIN_GETFUNC _IOWR(PTP_CLK_MAGIC, 6, struct ptp_pin_desc)
#define PTP_PIN_SETFUNC _IOW(PTP_CLK_MAGIC, 7, struct ptp_pin_desc)
#define PTP_SYS_OFFSET_PRECISE \
_IOWR(PTP_CLK_MAGIC, 8, struct ptp_sys_offset_precise)
struct ptp_extts_event {
struct ptp_clock_time t; /* Time event occured. */

292
kernel/time/timekeeping.c
View File

@ -233,6 +233,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
u64 tmp, ntpinterval;
struct clocksource *old_clock;
++tk->cs_was_changed_seq;
old_clock = tk->tkr_mono.clock;
tk->tkr_mono.clock = clock;
tk->tkr_mono.read = clock->read;
@ -298,19 +299,36 @@ u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
static inline u32 arch_gettimeoffset(void) { return 0; }
#endif
static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
static inline s64 timekeeping_delta_to_ns(struct tk_read_base *tkr,
cycle_t delta)
{
cycle_t delta;
s64 nsec;
delta = timekeeping_get_delta(tkr);
nsec = (delta * tkr->mult + tkr->xtime_nsec) >> tkr->shift;
nsec = delta * tkr->mult + tkr->xtime_nsec;
nsec >>= tkr->shift;
/* If arch requires, add in get_arch_timeoffset() */
return nsec + arch_gettimeoffset();
}
static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
{
cycle_t delta;
delta = timekeeping_get_delta(tkr);
return timekeeping_delta_to_ns(tkr, delta);
}
static inline s64 timekeeping_cycles_to_ns(struct tk_read_base *tkr,
cycle_t cycles)
{
cycle_t delta;
/* calculate the delta since the last update_wall_time */
delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
return timekeeping_delta_to_ns(tkr, delta);
}
/**
* update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
* @tkr: Timekeeping readout base from which we take the update
@ -857,44 +875,262 @@ time64_t __ktime_get_real_seconds(void)
return tk->xtime_sec;
}
#ifdef CONFIG_NTP_PPS
/**
* ktime_get_raw_and_real_ts64 - get day and raw monotonic time in timespec format
* @ts_raw: pointer to the timespec to be set to raw monotonic time
* @ts_real: pointer to the timespec to be set to the time of day
*
* This function reads both the time of day and raw monotonic time at the
* same time atomically and stores the resulting timestamps in timespec
* format.
* ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
* @systime_snapshot: pointer to struct receiving the system time snapshot
*/
void ktime_get_raw_and_real_ts64(struct timespec64 *ts_raw, struct timespec64 *ts_real)
void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned long seq;
s64 nsecs_raw, nsecs_real;
ktime_t base_raw;
ktime_t base_real;
s64 nsec_raw;
s64 nsec_real;
cycle_t now;
WARN_ON_ONCE(timekeeping_suspended);
do {
seq = read_seqcount_begin(&tk_core.seq);
*ts_raw = tk->raw_time;
ts_real->tv_sec = tk->xtime_sec;
ts_real->tv_nsec = 0;
nsecs_raw = timekeeping_get_ns(&tk->tkr_raw);
nsecs_real = timekeeping_get_ns(&tk->tkr_mono);
now = tk->tkr_mono.read(tk->tkr_mono.clock);
systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
base_real = ktime_add(tk->tkr_mono.base,
tk_core.timekeeper.offs_real);
base_raw = tk->tkr_raw.base;
nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
} while (read_seqcount_retry(&tk_core.seq, seq));
timespec64_add_ns(ts_raw, nsecs_raw);
timespec64_add_ns(ts_real, nsecs_real);
systime_snapshot->cycles = now;
systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
}
EXPORT_SYMBOL(ktime_get_raw_and_real_ts64);
EXPORT_SYMBOL_GPL(ktime_get_snapshot);
#endif /* CONFIG_NTP_PPS */
/* Scale base by mult/div checking for overflow */
static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
{
u64 tmp, rem;
tmp = div64_u64_rem(*base, div, &rem);
if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
return -EOVERFLOW;
tmp *= mult;
rem *= mult;
do_div(rem, div);
*base = tmp + rem;
return 0;
}
/**
* adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
* @history: Snapshot representing start of history
* @partial_history_cycles: Cycle offset into history (fractional part)
* @total_history_cycles: Total history length in cycles
* @discontinuity: True indicates clock was set on history period
* @ts: Cross timestamp that should be adjusted using
* partial/total ratio
*
* Helper function used by get_device_system_crosststamp() to correct the
* crosstimestamp corresponding to the start of the current interval to the
* system counter value (timestamp point) provided by the driver. The
* total_history_* quantities are the total history starting at the provided
* reference point and ending at the start of the current interval. The cycle
* count between the driver timestamp point and the start of the current
* interval is partial_history_cycles.
*/
static int adjust_historical_crosststamp(struct system_time_snapshot *history,
cycle_t partial_history_cycles,
cycle_t total_history_cycles,
bool discontinuity,
struct system_device_crosststamp *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
u64 corr_raw, corr_real;
bool interp_forward;
int ret;
if (total_history_cycles == 0 || partial_history_cycles == 0)
return 0;
/* Interpolate shortest distance from beginning or end of history */
interp_forward = partial_history_cycles > total_history_cycles/2 ?
true : false;
partial_history_cycles = interp_forward ?
total_history_cycles - partial_history_cycles :
partial_history_cycles;
/*
* Scale the monotonic raw time delta by:
* partial_history_cycles / total_history_cycles
*/
corr_raw = (u64)ktime_to_ns(
ktime_sub(ts->sys_monoraw, history->raw));
ret = scale64_check_overflow(partial_history_cycles,
total_history_cycles, &corr_raw);
if (ret)
return ret;
/*
* If there is a discontinuity in the history, scale monotonic raw
* correction by:
* mult(real)/mult(raw) yielding the realtime correction
* Otherwise, calculate the realtime correction similar to monotonic
* raw calculation
*/
if (discontinuity) {
corr_real = mul_u64_u32_div
(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
} else {
corr_real = (u64)ktime_to_ns(
ktime_sub(ts->sys_realtime, history->real));
ret = scale64_check_overflow(partial_history_cycles,
total_history_cycles, &corr_real);
if (ret)
return ret;
}
/* Fixup monotonic raw and real time time values */
if (interp_forward) {
ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
ts->sys_realtime = ktime_add_ns(history->real, corr_real);
} else {
ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
}
return 0;
}
/*
* cycle_between - true if test occurs chronologically between before and after
*/
static bool cycle_between(cycle_t before, cycle_t test, cycle_t after)
{
if (test > before && test < after)
return true;
if (test < before && before > after)
return true;
return false;
}
/**
* get_device_system_crosststamp - Synchronously capture system/device timestamp
* @get_time_fn: Callback to get simultaneous device time and
* system counter from the device driver
* @ctx: Context passed to get_time_fn()
* @history_begin: Historical reference point used to interpolate system
* time when counter provided by the driver is before the current interval
* @xtstamp: Receives simultaneously captured system and device time
*
* Reads a timestamp from a device and correlates it to system time
*/
int get_device_system_crosststamp(int (*get_time_fn)
(ktime_t *device_time,
struct system_counterval_t *sys_counterval,
void *ctx),
void *ctx,
struct system_time_snapshot *history_begin,
struct system_device_crosststamp *xtstamp)
{
struct system_counterval_t system_counterval;
struct timekeeper *tk = &tk_core.timekeeper;
cycle_t cycles, now, interval_start;
unsigned int clock_was_set_seq;
ktime_t base_real, base_raw;
s64 nsec_real, nsec_raw;
u8 cs_was_changed_seq;
unsigned long seq;
bool do_interp;
int ret;
do {
seq = read_seqcount_begin(&tk_core.seq);
/*
* Try to synchronously capture device time and a system
* counter value calling back into the device driver
*/
ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
if (ret)
return ret;
/*
* Verify that the clocksource associated with the captured
* system counter value is the same as the currently installed
* timekeeper clocksource
*/
if (tk->tkr_mono.clock != system_counterval.cs)
return -ENODEV;
cycles = system_counterval.cycles;
/*
* Check whether the system counter value provided by the
* device driver is on the current timekeeping interval.
*/
now = tk->tkr_mono.read(tk->tkr_mono.clock);
interval_start = tk->tkr_mono.cycle_last;
if (!cycle_between(interval_start, cycles, now)) {
clock_was_set_seq = tk->clock_was_set_seq;
cs_was_changed_seq = tk->cs_was_changed_seq;
cycles = interval_start;
do_interp = true;
} else {
do_interp = false;
}
base_real = ktime_add(tk->tkr_mono.base,
tk_core.timekeeper.offs_real);
base_raw = tk->tkr_raw.base;
nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono,
system_counterval.cycles);
nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw,
system_counterval.cycles);
} while (read_seqcount_retry(&tk_core.seq, seq));
xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
/*
* Interpolate if necessary, adjusting back from the start of the
* current interval
*/
if (do_interp) {
cycle_t partial_history_cycles, total_history_cycles;
bool discontinuity;
/*
* Check that the counter value occurs after the provided
* history reference and that the history doesn't cross a
* clocksource change
*/
if (!history_begin ||
!cycle_between(history_begin->cycles,
system_counterval.cycles, cycles) ||
history_begin->cs_was_changed_seq != cs_was_changed_seq)
return -EINVAL;
partial_history_cycles = cycles - system_counterval.cycles;
total_history_cycles = cycles - history_begin->cycles;
discontinuity =
history_begin->clock_was_set_seq != clock_was_set_seq;
ret = adjust_historical_crosststamp(history_begin,
partial_history_cycles,
total_history_cycles,
discontinuity, xtstamp);
if (ret)
return ret;
}
return 0;
}
EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
/**
* do_gettimeofday - Returns the time of day in a timeval