// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2015 Anton Ivanov (aivanov@{brocade.com,kot-begemot.co.uk}) * Copyright (C) 2015 Thomas Meyer (thomas@m3y3r.de) * Copyright (C) 2012-2014 Cisco Systems * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) * Copyright (C) 2019 Intel Corporation */ #include <linux/clockchips.h>#include <linux/init.h>#include <linux/interrupt.h>#include <linux/jiffies.h>#include <linux/mm.h>#include <linux/sched.h>#include <linux/spinlock.h>#include <linux/threads.h>#include <asm/irq.h>#include <asm/param.h>#include <kern_util.h>#include <os.h>#include <linux/delay.h>#include <linux/time-internal.h>#include <linux/um_timetravel.h>#include <shared/init.h> #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT#include <linux/sched/clock.h> enum time_travel_mode time_travel_mode;EXPORT_SYMBOL_GPL(time_travel_mode); static bool time_travel_start_set;static unsigned long long time_travel_start;static unsigned long long time_travel_time;static unsigned long long time_travel_shm_offset;static LIST_HEAD(time_travel_events);static LIST_HEAD(time_travel_irqs);static unsigned long long time_travel_timer_interval;static unsigned long long time_travel_next_event;static struct time_travel_event time_travel_timer_event;static int time_travel_ext_fd = -1;static unsigned int time_travel_ext_waiting;static bool time_travel_ext_prev_request_valid;static unsigned long long time_travel_ext_prev_request;static unsigned long long *time_travel_ext_free_until;static unsigned long long _time_travel_ext_free_until;static u16 time_travel_shm_id;static struct um_timetravel_schedshm *time_travel_shm;static union um_timetravel_schedshm_client *time_travel_shm_client; unsigned long tt_extra_sched_jiffies; notrace unsigned long long sched_clock(void){	return (unsigned long long)(jiffies - INITIAL_JIFFIES +				    tt_extra_sched_jiffies)					* (NSEC_PER_SEC / HZ);} static void time_travel_set_time(unsigned long long ns){	if (unlikely(ns < time_travel_time))		panic("time-travel: time goes backwards %lld -> %lld\n",		      time_travel_time, ns);	else if (unlikely(ns >= S64_MAX))		panic("The system was going to sleep forever, aborting"); 	time_travel_time = ns;} enum time_travel_message_handling {	TTMH_IDLE,	TTMH_POLL,	TTMH_READ,	TTMH_READ_START_ACK,}; static u64 bc_message;int time_travel_should_print_bc_msg; void _time_travel_print_bc_msg(void){	time_travel_should_print_bc_msg = 0;	printk(KERN_INFO "time-travel: received broadcast 0x%llx\n", bc_message);} static void time_travel_setup_shm(int fd, u16 id){	u32 len; 	time_travel_shm = os_mmap_rw_shared(fd, sizeof(*time_travel_shm)); 	if (!time_travel_shm)		goto out; 	len = time_travel_shm->len; 	if (time_travel_shm->version != UM_TIMETRAVEL_SCHEDSHM_VERSION ||	    len < struct_size(time_travel_shm, clients, id + 1)) {		os_unmap_memory(time_travel_shm, sizeof(*time_travel_shm));		time_travel_shm = NULL;		goto out;	} 	time_travel_shm = os_mremap_rw_shared(time_travel_shm,					      sizeof(*time_travel_shm),					      len);	if (!time_travel_shm)		goto out; 	time_travel_shm_offset = time_travel_shm->current_time;	time_travel_shm_client = &time_travel_shm->clients[id];	time_travel_shm_client->capa |= UM_TIMETRAVEL_SCHEDSHM_CAP_TIME_SHARE;	time_travel_shm_id = id;	/* always look at that free_until from now on */	time_travel_ext_free_until = &time_travel_shm->free_until;out:	os_close_file(fd);} static void time_travel_handle_message(struct um_timetravel_msg *msg,				       enum time_travel_message_handling mode){	struct um_timetravel_msg resp = {		.op = UM_TIMETRAVEL_ACK,	};	int ret; 	/*	 * We can't unlock here, but interrupt signals with a timetravel_handler	 * (see um_request_irq_tt) get to the timetravel_handler anyway.	 */	if (mode != TTMH_READ) {		BUG_ON(mode == TTMH_IDLE && !irqs_disabled()); 		while (os_poll(1, &time_travel_ext_fd) != 0) {			/* nothing */		}	} 	if (unlikely(mode == TTMH_READ_START_ACK)) {		int fd[UM_TIMETRAVEL_SHARED_MAX_FDS]; 		ret = os_rcv_fd_msg(time_travel_ext_fd, fd,				    ARRAY_SIZE(fd), msg, sizeof(*msg));		if (ret == sizeof(*msg)) {			time_travel_setup_shm(fd[UM_TIMETRAVEL_SHARED_MEMFD],					      msg->time & UM_TIMETRAVEL_START_ACK_ID);			/* we don't use the logging for now */			os_close_file(fd[UM_TIMETRAVEL_SHARED_LOGFD]);		}	} else {		ret = os_read_file(time_travel_ext_fd, msg, sizeof(*msg));	} 	if (ret == 0)		panic("time-travel external link is broken\n");	if (ret != sizeof(*msg))		panic("invalid time-travel message - %d bytes\n", ret); 	switch (msg->op) {	default:		WARN_ONCE(1, "time-travel: unexpected message %lld\n",			  (unsigned long long)msg->op);		break;	case UM_TIMETRAVEL_ACK:		return;	case UM_TIMETRAVEL_RUN:		time_travel_set_time(msg->time);		if (time_travel_shm) {			/* no request right now since we're running */			time_travel_shm_client->flags &=				~UM_TIMETRAVEL_SCHEDSHM_FLAGS_REQ_RUN;			/* no ack for shared memory RUN */			return;		}		break;	case UM_TIMETRAVEL_FREE_UNTIL:		/* not supposed to get this with shm, but ignore it */		if (time_travel_shm)			break;		time_travel_ext_free_until = &_time_travel_ext_free_until;		_time_travel_ext_free_until = msg->time;		break;	case UM_TIMETRAVEL_BROADCAST:		bc_message = msg->time;		time_travel_should_print_bc_msg = 1;		break;	} 	resp.seq = msg->seq;	os_write_file(time_travel_ext_fd, &resp, sizeof(resp));} static u64 time_travel_ext_req(u32 op, u64 time){	static int seq;	int mseq = ++seq;	struct um_timetravel_msg msg = {		.op = op,		.time = time,		.seq = mseq,	}; 	/*	 * We need to block even the timetravel handlers of SIGIO here and	 * only restore their use when we got the ACK - otherwise we may	 * (will) get interrupted by that, try to queue the IRQ for future	 * processing and thus send another request while we're still waiting	 * for an ACK, but the peer doesn't know we got interrupted and will	 * send the ACKs in the same order as the message, but we'd need to	 * see them in the opposite order ...	 *	 * This wouldn't matter *too* much, but some ACKs carry the	 * current time (for UM_TIMETRAVEL_GET) and getting another	 * ACK without a time would confuse us a lot!	 *	 * The sequence number assignment that happens here lets us	 * debug such message handling issues more easily.	 */	block_signals_hard();	os_write_file(time_travel_ext_fd, &msg, sizeof(msg)); 	/* no ACK expected for WAIT in shared memory mode */	if (msg.op == UM_TIMETRAVEL_WAIT && time_travel_shm)		goto done; 	while (msg.op != UM_TIMETRAVEL_ACK)		time_travel_handle_message(&msg,					   op == UM_TIMETRAVEL_START ?						TTMH_READ_START_ACK :						TTMH_READ); 	if (msg.seq != mseq)		panic("time-travel: ACK message has different seqno! op=%d, seq=%d != %d time=%lld\n",		      msg.op, msg.seq, mseq, msg.time); 	if (op == UM_TIMETRAVEL_GET)		time_travel_set_time(msg.time);done:	unblock_signals_hard(); 	return msg.time;} void __time_travel_wait_readable(int fd){	int fds[2] = { fd, time_travel_ext_fd };	int ret; 	if (time_travel_mode != TT_MODE_EXTERNAL)		return; 	while ((ret = os_poll(2, fds))) {		struct um_timetravel_msg msg; 		if (ret == 1)			time_travel_handle_message(&msg, TTMH_READ);	}}EXPORT_SYMBOL_GPL(__time_travel_wait_readable); static void time_travel_ext_update_request(unsigned long long time){	if (time_travel_mode != TT_MODE_EXTERNAL)		return; 	/* asked for exactly this time previously */	if (time_travel_ext_prev_request_valid &&	    time == time_travel_ext_prev_request)		return; 	/*	 * if we're running and are allowed to run past the request	 * then we don't need to update it either	 *	 * Note for shm we ignore FREE_UNTIL messages and leave the pointer	 * to shared memory, and for non-shm the offset is 0.	 */	if (!time_travel_ext_waiting && time_travel_ext_free_until &&	    time < (*time_travel_ext_free_until - time_travel_shm_offset))		return; 	time_travel_ext_prev_request = time;	time_travel_ext_prev_request_valid = true; 	if (time_travel_shm) {		union um_timetravel_schedshm_client *running; 		running = &time_travel_shm->clients[time_travel_shm->running_id]; 		if (running->capa & UM_TIMETRAVEL_SCHEDSHM_CAP_TIME_SHARE) {			time_travel_shm_client->flags |=				UM_TIMETRAVEL_SCHEDSHM_FLAGS_REQ_RUN;			time += time_travel_shm_offset;			time_travel_shm_client->req_time = time;			if (time < time_travel_shm->free_until)				time_travel_shm->free_until = time;			return;		}	} 	time_travel_ext_req(UM_TIMETRAVEL_REQUEST, time);} void __time_travel_propagate_time(void){	static unsigned long long last_propagated; 	if (time_travel_shm) {		if (time_travel_shm->running_id != time_travel_shm_id)			panic("time-travel: setting time while not running\n");		time_travel_shm->current_time = time_travel_time +						time_travel_shm_offset;		return;	} 	if (last_propagated == time_travel_time)		return; 	time_travel_ext_req(UM_TIMETRAVEL_UPDATE, time_travel_time);	last_propagated = time_travel_time;}EXPORT_SYMBOL_GPL(__time_travel_propagate_time); /* returns true if we must do a wait to the simtime device */static bool time_travel_ext_request(unsigned long long time){	/*	 * If we received an external sync point ("free until") then we	 * don't have to request/wait for anything until then, unless	 * we're already waiting.	 *	 * Note for shm we ignore FREE_UNTIL messages and leave the pointer	 * to shared memory, and for non-shm the offset is 0.	 */	if (!time_travel_ext_waiting && time_travel_ext_free_until &&	    time < (*time_travel_ext_free_until - time_travel_shm_offset))		return false; 	time_travel_ext_update_request(time);	return true;} static void time_travel_ext_wait(bool idle){	struct um_timetravel_msg msg = {		.op = UM_TIMETRAVEL_ACK,	}; 	time_travel_ext_prev_request_valid = false;	if (!time_travel_shm)		time_travel_ext_free_until = NULL;	time_travel_ext_waiting++; 	time_travel_ext_req(UM_TIMETRAVEL_WAIT, -1); 	/*	 * Here we are deep in the idle loop, so we have to break out of the	 * kernel abstraction in a sense and implement this in terms of the	 * UML system waiting on the VQ interrupt while sleeping, when we get	 * the signal it'll call time_travel_ext_vq_notify_done() completing the	 * call.	 */	while (msg.op != UM_TIMETRAVEL_RUN)		time_travel_handle_message(&msg, idle ? TTMH_IDLE : TTMH_POLL); 	time_travel_ext_waiting--; 	/* we might request more stuff while polling - reset when we run */	time_travel_ext_prev_request_valid = false;} static void time_travel_ext_get_time(void){	if (time_travel_shm)		time_travel_set_time(time_travel_shm->current_time -				     time_travel_shm_offset);	else		time_travel_ext_req(UM_TIMETRAVEL_GET, -1);} static void __time_travel_update_time(unsigned long long ns, bool idle){	if (time_travel_mode == TT_MODE_EXTERNAL && time_travel_ext_request(ns))		time_travel_ext_wait(idle);	else		time_travel_set_time(ns);} static struct time_travel_event *time_travel_first_event(void){	return list_first_entry_or_null(&time_travel_events,					struct time_travel_event,					list);} static void __time_travel_add_event(struct time_travel_event *e,				    unsigned long long time){	struct time_travel_event *tmp;	bool inserted = false;	unsigned long flags; 	if (e->pending)		return; 	e->pending = true;	e->time = time; 	local_irq_save(flags);	list_for_each_entry(tmp, &time_travel_events, list) {		/*		 * Add the new entry before one with higher time,		 * or if they're equal and both on stack, because		 * in that case we need to unwind the stack in the		 * right order, and the later event (timer sleep		 * or such) must be dequeued first.		 */		if ((tmp->time > e->time) ||		    (tmp->time == e->time && tmp->onstack && e->onstack)) {			list_add_tail(&e->list, &tmp->list);			inserted = true;			break;		}	} 	if (!inserted)		list_add_tail(&e->list, &time_travel_events); 	tmp = time_travel_first_event();	time_travel_ext_update_request(tmp->time);	time_travel_next_event = tmp->time;	local_irq_restore(flags);} static void time_travel_add_event(struct time_travel_event *e,				  unsigned long long time){	if (WARN_ON(!e->fn))		return; 	__time_travel_add_event(e, time);} void time_travel_add_event_rel(struct time_travel_event *e,			       unsigned long long delay_ns){	time_travel_add_event(e, time_travel_time + delay_ns);} static void time_travel_periodic_timer(struct time_travel_event *e){	time_travel_add_event(&time_travel_timer_event,			      time_travel_time + time_travel_timer_interval); 	/* clock tick; decrease extra jiffies by keeping sched_clock constant */	if (tt_extra_sched_jiffies > 0)		tt_extra_sched_jiffies -= 1; 	deliver_alarm();} void deliver_time_travel_irqs(void){	struct time_travel_event *e;	unsigned long flags; 	/*	 * Don't do anything for most cases. Note that because here we have	 * to disable IRQs (and re-enable later) we'll actually recurse at	 * the end of the function, so this is strictly necessary.	 */	if (likely(list_empty(&time_travel_irqs)))		return; 	local_irq_save(flags);	irq_enter();	while ((e = list_first_entry_or_null(&time_travel_irqs,					     struct time_travel_event,					     list))) {		list_del(&e->list);		e->pending = false;		e->fn(e);	}	irq_exit();	local_irq_restore(flags);} static void time_travel_deliver_event(struct time_travel_event *e){	if (e == &time_travel_timer_event) {		/*		 * deliver_alarm() does the irq_enter/irq_exit		 * by itself, so must handle it specially here		 */		e->fn(e);	} else if (irqs_disabled()) {		list_add_tail(&e->list, &time_travel_irqs);

Lines 1–500 of 1092. Load more