SPI is one of the interfaces used to access devices which have a POSIX clock driver (real time clocks, 1588 timers etc). The fact that the SPI bus is slow is not what the main problem is, but rather the fact that drivers don't take a constant amount of time in transferring data over SPI. When there is a high delay in the readout of time, there will be uncertainty in the value that has been read out of the peripheral. When that delay is constant, the uncertainty can at least be approximated with a certain accuracy which is fine more often than not. Timing jitter occurs all over in the kernel code, and is mainly caused by having to let go of the CPU for various reasons such as preemption, servicing interrupts, going to sleep, etc. Another major reason is CPU dynamic frequency scaling. It turns out that the problem of retrieving time from a SPI peripheral with high accuracy can be solved by the use of "PTP system timestamping" - a mechanism to correlate the time when the device has snapshotted its internal time counter with the Linux system time at that same moment. This is sufficient for having a precise time measurement - it is not necessary for the whole SPI transfer to be transmitted "as fast as possible", or "as low-jitter as possible". The system has to be low-jitter for a very short amount of time to be effective. This patch introduces a PTP system timestamping mechanism in struct spi_transfer. This is to be used by SPI device drivers when they need to know the exact time at which the underlying device's time was snapshotted. More often than not, SPI peripherals have a very exact timing for when their SPI-to-interconnect bridge issues a transaction for snapshotting and reading the time register, and that will be dependent on when the SPI-to-interconnect bridge figures out that this is what it should do, aka as soon as it sees byte N of the SPI transfer. Since spi_device drivers are the ones who'd know best how the peripheral behaves in this regard, expose a mechanism in spi_transfer which allows them to specify which word (or word range) from the transfer should be timestamped. Add a default implementation of the PTP system timestamping in the SPI core. This is not going to be satisfactory performance-wise, but should at least increase the likelihood that SPI device drivers will use PTP system timestamping in the future. There are 3 entry points from the core towards the SPI controller drivers: - transfer_one: The driver is passed individual spi_transfers to execute. This is the easiest to timestamp. - transfer_one_message: The core passes the driver an entire spi_message (a potential batch of spi_transfers). The core puts the same pre and post timestamp to all transfers within a message. This is not ideal, but nothing better can be done by default anyway, since the core has no insight into how the driver batches the transfers. - transfer: Like transfer_one_message, but for unqueued drivers (i.e. the driver implements its own queue scheduling).
Unified · Split
| @@ -1171,6 +1171,11 @@ static int spi_transfer_one_message(struct spi_controller *ctlr, · 1170 unmodified lines | |||
| 1171 | spi_statistics_add_transfer_stats(statm, xfer, ctlr); | 1171 | spi_statistics_add_transfer_stats(statm, xfer, ctlr); |
| 1172 | spi_statistics_add_transfer_stats(stats, xfer, ctlr); | 1172 | spi_statistics_add_transfer_stats(stats, xfer, ctlr); |
| 1173 | 1173 | ||
| 1174 | if (!ctlr->ptp_sts_supported) { | ||
| 1175 | xfer->ptp_sts_word_pre = 0; | ||
| 1176 | ptp_read_system_prets(xfer->ptp_sts); | ||
| 1177 | } | ||
| 1178 | |||
| 1174 | if (xfer->tx_buf || xfer->rx_buf) { | 1179 | if (xfer->tx_buf || xfer->rx_buf) { |
| 1175 | reinit_completion(&ctlr->xfer_completion); | 1180 | reinit_completion(&ctlr->xfer_completion); |
| 1176 | 1181 | ||
| @@ -1197,6 +1202,11 @@ static int spi_transfer_one_message(struct spi_controller *ctlr, · 20 unmodified lines | |||
| 1197 | xfer->len); | 1202 | xfer->len); |
| 1198 | } | 1203 | } |
| 1199 | 1204 | ||
| 1205 | if (!ctlr->ptp_sts_supported) { | ||
| 1206 | ptp_read_system_postts(xfer->ptp_sts); | ||
| 1207 | xfer->ptp_sts_word_post = xfer->len; | ||
| 1208 | } | ||
| 1209 | |||
| 1200 | trace_spi_transfer_stop(msg, xfer); | 1210 | trace_spi_transfer_stop(msg, xfer); |
| 1201 | 1211 | ||
| 1202 | if (msg->status != -EINPROGRESS) | 1212 | if (msg->status != -EINPROGRESS) |
| @@ -1265,6 +1275,7 @@ EXPORT_SYMBOL_GPL(spi_finalize_current_transfer); · 62 unmodified lines | |||
| 1265 | */ | 1275 | */ |
| 1266 | static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) | 1276 | static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) |
| 1267 | { | 1277 | { |
| 1278 | struct spi_transfer *xfer; | ||
| 1268 | struct spi_message *msg; | 1279 | struct spi_message *msg; |
| 1269 | bool was_busy = false; | 1280 | bool was_busy = false; |
| 1270 | unsigned long flags; | 1281 | unsigned long flags; |
| @@ -1391,6 +1402,13 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) · 120 unmodified lines | |||
| 1391 | goto out; | 1402 | goto out; |
| 1392 | } | 1403 | } |
| 1393 | 1404 | ||
| 1405 | if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) { | ||
| 1406 | list_for_each_entry(xfer, &msg->transfers, transfer_list) { | ||
| 1407 | xfer->ptp_sts_word_pre = 0; | ||
| 1408 | ptp_read_system_prets(xfer->ptp_sts); | ||
| 1409 | } | ||
| 1410 | } | ||
| 1411 | |||
| 1394 | ret = ctlr->transfer_one_message(ctlr, msg); | 1412 | ret = ctlr->transfer_one_message(ctlr, msg); |
| 1395 | if (ret) { | 1413 | if (ret) { |
| 1396 | dev_err(&ctlr->dev, | 1414 | dev_err(&ctlr->dev, |
| @@ -1418,6 +1436,99 @@ static void spi_pump_messages(struct kthread_work *work) · 21 unmodified lines | |||
| 1418 | __spi_pump_messages(ctlr, true); | 1436 | __spi_pump_messages(ctlr, true); |
| 1419 | } | 1437 | } |
| 1420 | 1438 | ||
| 1439 | /** | ||
| 1440 | * spi_take_timestamp_pre - helper for drivers to collect the beginning of the | ||
| 1441 | * TX timestamp for the requested byte from the SPI | ||
| 1442 | * transfer. The frequency with which this function | ||
| 1443 | * must be called (once per word, once for the whole | ||
| 1444 | * transfer, once per batch of words etc) is arbitrary | ||
| 1445 | * as long as the @tx buffer offset is greater than or | ||
| 1446 | * equal to the requested byte at the time of the | ||
| 1447 | * call. The timestamp is only taken once, at the | ||
| 1448 | * first such call. It is assumed that the driver | ||
| 1449 | * advances its @tx buffer pointer monotonically. | ||
| 1450 | * @ctlr: Pointer to the spi_controller structure of the driver | ||
| 1451 | * @xfer: Pointer to the transfer being timestamped | ||
| 1452 | * @tx: Pointer to the current word within the xfer->tx_buf that the driver is | ||
| 1453 | * preparing to transmit right now. | ||
| 1454 | * @irqs_off: If true, will disable IRQs and preemption for the duration of the | ||
| 1455 | * transfer, for less jitter in time measurement. Only compatible | ||
| 1456 | * with PIO drivers. If true, must follow up with | ||
| 1457 | * spi_take_timestamp_post or otherwise system will crash. | ||
| 1458 | * WARNING: for fully predictable results, the CPU frequency must | ||
| 1459 | * also be under control (governor). | ||
| 1460 | */ | ||
| 1461 | void spi_take_timestamp_pre(struct spi_controller *ctlr, | ||
| 1462 | struct spi_transfer *xfer, | ||
| 1463 | const void *tx, bool irqs_off) | ||
| 1464 | { | ||
| 1465 | u8 bytes_per_word = DIV_ROUND_UP(xfer->bits_per_word, 8); | ||
| 1466 | |||
| 1467 | if (!xfer->ptp_sts) | ||
| 1468 | return; | ||
| 1469 | |||
| 1470 | if (xfer->timestamped_pre) | ||
| 1471 | return; | ||
| 1472 | |||
| 1473 | if (tx < (xfer->tx_buf + xfer->ptp_sts_word_pre * bytes_per_word)) | ||
| 1474 | return; | ||
| 1475 | |||
| 1476 | /* Capture the resolution of the timestamp */ | ||
| 1477 | xfer->ptp_sts_word_pre = (tx - xfer->tx_buf) / bytes_per_word; | ||
| 1478 | |||
| 1479 | xfer->timestamped_pre = true; | ||
| 1480 | |||
| 1481 | if (irqs_off) { | ||
| 1482 | local_irq_save(ctlr->irq_flags); | ||
| 1483 | preempt_disable(); | ||
| 1484 | } | ||
| 1485 | |||
| 1486 | ptp_read_system_prets(xfer->ptp_sts); | ||
| 1487 | } | ||
| 1488 | EXPORT_SYMBOL_GPL(spi_take_timestamp_pre); | ||
| 1489 | |||
| 1490 | /** | ||
| 1491 | * spi_take_timestamp_post - helper for drivers to collect the end of the | ||
| 1492 | * TX timestamp for the requested byte from the SPI | ||
| 1493 | * transfer. Can be called with an arbitrary | ||
| 1494 | * frequency: only the first call where @tx exceeds | ||
| 1495 | * or is equal to the requested word will be | ||
| 1496 | * timestamped. | ||
| 1497 | * @ctlr: Pointer to the spi_controller structure of the driver | ||
| 1498 | * @xfer: Pointer to the transfer being timestamped | ||
| 1499 | * @tx: Pointer to the current word within the xfer->tx_buf that the driver has | ||
| 1500 | * just transmitted. | ||
| 1501 | * @irqs_off: If true, will re-enable IRQs and preemption for the local CPU. | ||
| 1502 | */ | ||
| 1503 | void spi_take_timestamp_post(struct spi_controller *ctlr, | ||
| 1504 | struct spi_transfer *xfer, | ||
| 1505 | const void *tx, bool irqs_off) | ||
| 1506 | { | ||
| 1507 | u8 bytes_per_word = DIV_ROUND_UP(xfer->bits_per_word, 8); | ||
| 1508 | |||
| 1509 | if (!xfer->ptp_sts) | ||
| 1510 | return; | ||
| 1511 | |||
| 1512 | if (xfer->timestamped_post) | ||
| 1513 | return; | ||
| 1514 | |||
| 1515 | if (tx < (xfer->tx_buf + xfer->ptp_sts_word_post * bytes_per_word)) | ||
| 1516 | return; | ||
| 1517 | |||
| 1518 | ptp_read_system_postts(xfer->ptp_sts); | ||
| 1519 | |||
| 1520 | if (irqs_off) { | ||
| 1521 | local_irq_restore(ctlr->irq_flags); | ||
| 1522 | preempt_enable(); | ||
| 1523 | } | ||
| 1524 | |||
| 1525 | /* Capture the resolution of the timestamp */ | ||
| 1526 | xfer->ptp_sts_word_post = (tx - xfer->tx_buf) / bytes_per_word; | ||
| 1527 | |||
| 1528 | xfer->timestamped_post = true; | ||
| 1529 | } | ||
| 1530 | EXPORT_SYMBOL_GPL(spi_take_timestamp_post); | ||
| 1531 | |||
| 1421 | /** | 1532 | /** |
| 1422 | * spi_set_thread_rt - set the controller to pump at realtime priority | 1533 | * spi_set_thread_rt - set the controller to pump at realtime priority |
| 1423 | * @ctlr: controller to boost priority of | 1534 | * @ctlr: controller to boost priority of |
| @@ -1503,6 +1614,7 @@ EXPORT_SYMBOL_GPL(spi_get_next_queued_message); · 79 unmodified lines | |||
| 1503 | */ | 1614 | */ |
| 1504 | void spi_finalize_current_message(struct spi_controller *ctlr) | 1615 | void spi_finalize_current_message(struct spi_controller *ctlr) |
| 1505 | { | 1616 | { |
| 1617 | struct spi_transfer *xfer; | ||
| 1506 | struct spi_message *mesg; | 1618 | struct spi_message *mesg; |
| 1507 | unsigned long flags; | 1619 | unsigned long flags; |
| 1508 | int ret; | 1620 | int ret; |
| @@ -1511,6 +1623,13 @@ void spi_finalize_current_message(struct spi_controller *ctlr) · 2 unmodified lines | |||
| 1511 | mesg = ctlr->cur_msg; | 1623 | mesg = ctlr->cur_msg; |
| 1512 | spin_unlock_irqrestore(&ctlr->queue_lock, flags); | 1624 | spin_unlock_irqrestore(&ctlr->queue_lock, flags); |
| 1513 | 1625 | ||
| 1626 | if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) { | ||
| 1627 | list_for_each_entry(xfer, &mesg->transfers, transfer_list) { | ||
| 1628 | ptp_read_system_postts(xfer->ptp_sts); | ||
| 1629 | xfer->ptp_sts_word_post = xfer->len; | ||
| 1630 | } | ||
| 1631 | } | ||
| 1632 | |||
| 1514 | spi_unmap_msg(ctlr, mesg); | 1633 | spi_unmap_msg(ctlr, mesg); |
| 1515 | 1634 | ||
| 1516 | if (ctlr->cur_msg_prepared && ctlr->unprepare_message) { | 1635 | if (ctlr->cur_msg_prepared && ctlr->unprepare_message) { |
| @@ -3273,6 +3392,7 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) · 1756 unmodified lines | |||
| 3273 | static int __spi_async(struct spi_device *spi, struct spi_message *message) | 3392 | static int __spi_async(struct spi_device *spi, struct spi_message *message) |
| 3274 | { | 3393 | { |
| 3275 | struct spi_controller *ctlr = spi->controller; | 3394 | struct spi_controller *ctlr = spi->controller; |
| 3395 | struct spi_transfer *xfer; | ||
| 3276 | 3396 | ||
| 3277 | /* | 3397 | /* |
| 3278 | * Some controllers do not support doing regular SPI transfers. Return | 3398 | * Some controllers do not support doing regular SPI transfers. Return |
| @@ -3288,6 +3408,13 @@ static int __spi_async(struct spi_device *spi, struct spi_message *message) · 9 unmodified lines | |||
| 3288 | 3408 | ||
| 3289 | trace_spi_message_submit(message); | 3409 | trace_spi_message_submit(message); |
| 3290 | 3410 | ||
| 3411 | if (!ctlr->ptp_sts_supported) { | ||
| 3412 | list_for_each_entry(xfer, &message->transfers, transfer_list) { | ||
| 3413 | xfer->ptp_sts_word_pre = 0; | ||
| 3414 | ptp_read_system_prets(xfer->ptp_sts); | ||
| 3415 | } | ||
| 3416 | } | ||
| 3417 | |||
| 3291 | return ctlr->transfer(spi, message); | 3418 | return ctlr->transfer(spi, message); |
| 3292 | } | 3419 | } |
| 3293 | 3420 | ||
| @@ -13,6 +13,7 @@ · 12 unmodified lines | |||
| 13 | #include <linux/completion.h> | 13 | #include <linux/completion.h> |
| 14 | #include <linux/scatterlist.h> | 14 | #include <linux/scatterlist.h> |
| 15 | #include <linux/gpio/consumer.h> | 15 | #include <linux/gpio/consumer.h> |
| 16 | #include <linux/ptp_clock_kernel.h> | ||
| 16 | 17 | ||
| 17 | struct dma_chan; | 18 | struct dma_chan; |
| 18 | struct property_entry; | 19 | struct property_entry; |
| @@ -409,6 +410,12 @@ static inline void spi_unregister_driver(struct spi_driver *sdrv) · 390 unmodified lines | |||
| 409 | * @fw_translate_cs: If the boot firmware uses different numbering scheme | 410 | * @fw_translate_cs: If the boot firmware uses different numbering scheme |
| 410 | * what Linux expects, this optional hook can be used to translate | 411 | * what Linux expects, this optional hook can be used to translate |
| 411 | * between the two. | 412 | * between the two. |
| 413 | * @ptp_sts_supported: If the driver sets this to true, it must provide a | ||
| 414 | * time snapshot in @spi_transfer->ptp_sts as close as possible to the | ||
| 415 | * moment in time when @spi_transfer->ptp_sts_word_pre and | ||
| 416 | * @spi_transfer->ptp_sts_word_post were transmitted. | ||
| 417 | * If the driver does not set this, the SPI core takes the snapshot as | ||
| 418 | * close to the driver hand-over as possible. | ||
| 412 | * | 419 | * |
| 413 | * Each SPI controller can communicate with one or more @spi_device | 420 | * Each SPI controller can communicate with one or more @spi_device |
| 414 | * children. These make a small bus, sharing MOSI, MISO and SCK signals | 421 | * children. These make a small bus, sharing MOSI, MISO and SCK signals |
| @@ -604,6 +611,15 @@ struct spi_controller { · 189 unmodified lines | |||
| 604 | void *dummy_tx; | 611 | void *dummy_tx; |
| 605 | 612 | ||
| 606 | int (*fw_translate_cs)(struct spi_controller *ctlr, unsigned cs); | 613 | int (*fw_translate_cs)(struct spi_controller *ctlr, unsigned cs); |
| 614 | |||
| 615 | /* | ||
| 616 | * Driver sets this field to indicate it is able to snapshot SPI | ||
| 617 | * transfers (needed e.g. for reading the time of POSIX clocks) | ||
| 618 | */ | ||
| 619 | bool ptp_sts_supported; | ||
| 620 | |||
| 621 | /* Interrupt enable state during PTP system timestamping */ | ||
| 622 | unsigned long irq_flags; | ||
| 607 | }; | 623 | }; |
| 608 | 624 | ||
| 609 | static inline void *spi_controller_get_devdata(struct spi_controller *ctlr) | 625 | static inline void *spi_controller_get_devdata(struct spi_controller *ctlr) |
| @@ -644,6 +660,14 @@ extern struct spi_message *spi_get_next_queued_message(struct spi_controller *ct · 34 unmodified lines | |||
| 644 | extern void spi_finalize_current_message(struct spi_controller *ctlr); | 660 | extern void spi_finalize_current_message(struct spi_controller *ctlr); |
| 645 | extern void spi_finalize_current_transfer(struct spi_controller *ctlr); | 661 | extern void spi_finalize_current_transfer(struct spi_controller *ctlr); |
| 646 | 662 | ||
| 663 | /* Helper calls for driver to timestamp transfer */ | ||
| 664 | void spi_take_timestamp_pre(struct spi_controller *ctlr, | ||
| 665 | struct spi_transfer *xfer, | ||
| 666 | const void *tx, bool irqs_off); | ||
| 667 | void spi_take_timestamp_post(struct spi_controller *ctlr, | ||
| 668 | struct spi_transfer *xfer, | ||
| 669 | const void *tx, bool irqs_off); | ||
| 670 | |||
| 647 | /* the spi driver core manages memory for the spi_controller classdev */ | 671 | /* the spi driver core manages memory for the spi_controller classdev */ |
| 648 | extern struct spi_controller *__spi_alloc_controller(struct device *host, | 672 | extern struct spi_controller *__spi_alloc_controller(struct device *host, |
| 649 | unsigned int size, bool slave); | 673 | unsigned int size, bool slave); |
| @@ -753,6 +777,35 @@ extern void spi_res_release(struct spi_controller *ctlr, · 103 unmodified lines | |||
| 753 | * @transfer_list: transfers are sequenced through @spi_message.transfers | 777 | * @transfer_list: transfers are sequenced through @spi_message.transfers |
| 754 | * @tx_sg: Scatterlist for transmit, currently not for client use | 778 | * @tx_sg: Scatterlist for transmit, currently not for client use |
| 755 | * @rx_sg: Scatterlist for receive, currently not for client use | 779 | * @rx_sg: Scatterlist for receive, currently not for client use |
| 780 | * @ptp_sts_word_pre: The word (subject to bits_per_word semantics) offset | ||
| 781 | * within @tx_buf for which the SPI device is requesting that the time | ||
| 782 | * snapshot for this transfer begins. Upon completing the SPI transfer, | ||
| 783 | * this value may have changed compared to what was requested, depending | ||
| 784 | * on the available snapshotting resolution (DMA transfer, | ||
| 785 | * @ptp_sts_supported is false, etc). | ||
| 786 | * @ptp_sts_word_post: See @ptp_sts_word_post. The two can be equal (meaning | ||
| 787 | * that a single byte should be snapshotted). | ||
| 788 | * If the core takes care of the timestamp (if @ptp_sts_supported is false | ||
| 789 | * for this controller), it will set @ptp_sts_word_pre to 0, and | ||
| 790 | * @ptp_sts_word_post to the length of the transfer. This is done | ||
| 791 | * purposefully (instead of setting to spi_transfer->len - 1) to denote | ||
| 792 | * that a transfer-level snapshot taken from within the driver may still | ||
| 793 | * be of higher quality. | ||
| 794 | * @ptp_sts: Pointer to a memory location held by the SPI slave device where a | ||
| 795 | * PTP system timestamp structure may lie. If drivers use PIO or their | ||
| 796 | * hardware has some sort of assist for retrieving exact transfer timing, | ||
| 797 | * they can (and should) assert @ptp_sts_supported and populate this | ||
| 798 | * structure using the ptp_read_system_*ts helper functions. | ||
| 799 | * The timestamp must represent the time at which the SPI slave device has | ||
| 800 | * processed the word, i.e. the "pre" timestamp should be taken before | ||
| 801 | * transmitting the "pre" word, and the "post" timestamp after receiving | ||
| 802 | * transmit confirmation from the controller for the "post" word. | ||
| 803 | * @timestamped_pre: Set by the SPI controller driver to denote it has acted | ||
| 804 | * upon the @ptp_sts request. Not set when the SPI core has taken care of | ||
| 805 | * the task. SPI device drivers are free to print a warning if this comes | ||
| 806 | * back unset and they need the better resolution. | ||
| 807 | * @timestamped_post: See above. The reason why both exist is that these | ||
| 808 | * booleans are also used to keep state in the core SPI logic. | ||
| 756 | * | 809 | * |
| 757 | * SPI transfers always write the same number of bytes as they read. | 810 | * SPI transfers always write the same number of bytes as they read. |
| 758 | * Protocol drivers should always provide @rx_buf and/or @tx_buf. | 811 | * Protocol drivers should always provide @rx_buf and/or @tx_buf. |
| @@ -842,6 +895,14 @@ struct spi_transfer { · 83 unmodified lines | |||
| 842 | 895 | ||
| 843 | u32 effective_speed_hz; | 896 | u32 effective_speed_hz; |
| 844 | 897 | ||
| 898 | unsigned int ptp_sts_word_pre; | ||
| 899 | unsigned int ptp_sts_word_post; | ||
| 900 | |||
| 901 | struct ptp_system_timestamp *ptp_sts; | ||
| 902 | |||
| 903 | bool timestamped_pre; | ||
| 904 | bool timestamped_post; | ||
| 905 | |||
| 845 | struct list_head transfer_list; | 906 | struct list_head transfer_list; |
| 846 | }; | 907 | }; |
| 847 | 908 | ||