Merge tag 'stable/for-linus-3.20-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip

Pull xen features and fixes from David Vrabel: - Reworked handling for foreign (grant mapped) pages to simplify the code, enable a number of additional use cases and fix a number of long-standing bugs. - Prefer the TSC over the Xen PV clock when dom0 (and the TSC is stable). - Assorted other cleanup and minor bug fixes. * tag 'stable/for-linus-3.20-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip: (25 commits) xen/manage: Fix USB interaction issues when resuming xenbus: Add proper handling of XS_ERROR from Xenbus for transactions. xen/gntdev: provide find_special_page VMA operation xen/gntdev: mark userspace PTEs as special on x86 PV guests xen-blkback: safely unmap grants in case they are still in use xen/gntdev: safely unmap grants in case they are still in use xen/gntdev: convert priv->lock to a mutex xen/grant-table: add a mechanism to safely unmap pages that are in use xen-netback: use foreign page information from the pages themselves xen: mark grant mapped pages as foreign xen/grant-table: add helpers for allocating pages x86/xen: require ballooned pages for grant maps xen: remove scratch frames for ballooned pages and m2p override xen/grant-table: pre-populate kernel unmap ops for xen_gnttab_unmap_refs() mm: add 'foreign' alias for the 'pinned' page flag mm: provide a find_special_page vma operation x86/xen: cleanup arch/x86/xen/mmu.c x86/xen: add some __init annotations in arch/x86/xen/mmu.c x86/xen: add some __init and static annotations in arch/x86/xen/setup.c x86/xen: use correct types for addresses in arch/x86/xen/setup.c ...

Linus Torvalds committed 11 years ago

Commit bdccc4e2 parents 98368ab 72978b228 files · +509 −594Browse files

Unified · Split

28 files

arch/arm/include/asm/xen/page.h +1 −1
arch/arm/xen/enlighten.c +2 −2
arch/arm/xen/mm.c +1 −1
arch/arm/xen/p2m.c +1 −1
arch/x86/include/asm/xen/page.h +5 −15
arch/x86/xen/mmu.c +8 −9
arch/x86/xen/p2m.c +11 −256
arch/x86/xen/setup.c +14 −23
arch/x86/xen/smp.c +1 −1
arch/x86/xen/time.c +4 −0
arch/x86/xen/xen-ops.h +6 −0
drivers/block/xen-blkback/blkback.c +123 −54
drivers/block/xen-blkback/common.h +3 −0
drivers/net/xen-netback/interface.c +3 −4
drivers/net/xen-netback/netback.c +9 −97
drivers/xen/balloon.c +2 −84
drivers/xen/gntdev.c +111 −32
drivers/xen/grant-table.c +116 −4
drivers/xen/manage.c +7 −1
drivers/xen/tmem.c +1 −1
drivers/xen/xen-scsiback.c +3 −3
drivers/xen/xenbus/xenbus_dev_frontend.c +7 −4
include/linux/mm.h +8 −0
include/linux/page-flags.h +5 −0
include/xen/grant_table.h +42 −1
include/xen/interface/features.h +6 −0
include/xen/interface/grant_table.h +7 −0
mm/memory.c +2 −0

arch/arm/include/asm/xen/page.h+1−1

@@ -92,7 +92,7 @@ extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, · 91 unmodified lines
92	92	struct page **pages, unsigned int count);
93	93
94	94	extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
95		− struct gnttab_map_grant_ref *kmap_ops,
	95	+ struct gnttab_unmap_grant_ref *kunmap_ops,
96	96	struct page **pages, unsigned int count);
97	97
98	98	bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn);

arch/arm/xen/enlighten.c+2−2

@@ -29,10 +29,10 @@ · 28 unmodified lines
29	29
30	30	struct start_info _xen_start_info;
31	31	struct start_info *xen_start_info = &_xen_start_info;
32		−EXPORT_SYMBOL_GPL(xen_start_info);
	32	+EXPORT_SYMBOL(xen_start_info);
33	33
34	34	enum xen_domain_type xen_domain_type = XEN_NATIVE;
35		−EXPORT_SYMBOL_GPL(xen_domain_type);
	35	+EXPORT_SYMBOL(xen_domain_type);
36	36
37	37	struct shared_info xen_dummy_shared_info;
38	38	struct shared_info HYPERVISOR_shared_info = (void )&xen_dummy_shared_info;

arch/arm/xen/mm.c+1−1

@@ -149,7 +149,7 @@ void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order) · 148 unmodified lines
149	149	EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
150	150
151	151	struct dma_map_ops *xen_dma_ops;
152		−EXPORT_SYMBOL_GPL(xen_dma_ops);
	152	+EXPORT_SYMBOL(xen_dma_ops);
153	153
154	154	static struct dma_map_ops xen_swiotlb_dma_ops = {
155	155	.mapping_error = xen_swiotlb_dma_mapping_error,

arch/arm/xen/p2m.c+1−1

@@ -102,7 +102,7 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, · 101 unmodified lines
102	102	EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping);
103	103
104	104	int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
105		− struct gnttab_map_grant_ref *kmap_ops,
	105	+ struct gnttab_unmap_grant_ref *kunmap_ops,
106	106	struct page **pages, unsigned int count)
107	107	{
108	108	int i;

arch/x86/include/asm/xen/page.h+5−15

@@ -55,9 +55,8 @@ extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, · 54 unmodified lines
55	55	struct gnttab_map_grant_ref *kmap_ops,
56	56	struct page **pages, unsigned int count);
57	57	extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
58		− struct gnttab_map_grant_ref *kmap_ops,
	58	+ struct gnttab_unmap_grant_ref *kunmap_ops,
59	59	struct page **pages, unsigned int count);
60		−extern unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn);
61	60
62	61	/*
63	62	* Helper functions to write or read unsigned long values to/from
@@ -154,21 +153,12 @@ static inline unsigned long mfn_to_pfn(unsigned long mfn) · 90 unmodified lines
154	153	return mfn;
155	154
156	155	pfn = mfn_to_pfn_no_overrides(mfn);
157		− if (__pfn_to_mfn(pfn) != mfn) {
158		− /*
159		− * If this appears to be a foreign mfn (because the pfn
160		− * doesn't map back to the mfn), then check the local override
161		− * table to see if there's a better pfn to use.
162		− *
163		− * m2p_find_override_pfn returns ~0 if it doesn't find anything.
164		− */
165		− pfn = m2p_find_override_pfn(mfn, ~0);
166		− }
	156	+ if (__pfn_to_mfn(pfn) != mfn)
	157	+ pfn = ~0;
167	158
168	159	/*
169		− * pfn is ~0 if there are no entries in the m2p for mfn or if the
170		− * entry doesn't map back to the mfn and m2p_override doesn't have a
171		− * valid entry for it.
	160	+ * pfn is ~0 if there are no entries in the m2p for mfn or the
	161	+ * entry doesn't map back to the mfn.
172	162	*/
173	163	if (pfn == ~0 && __pfn_to_mfn(mfn) == IDENTITY_FRAME(mfn))
174	164	pfn = mfn;

arch/x86/xen/mmu.c+8−9

@@ -1489,7 +1489,7 @@ static void __init xen_set_pte_init(pte_t *ptep, pte_t pte) · 1488 unmodified lines
1489	1489	native_set_pte(ptep, pte);
1490	1490	}
1491	1491
1492		−static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
	1492	+static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1493	1493	{
1494	1494	struct mmuext_op op;
1495	1495	op.cmd = cmd;
@@ -1657,7 +1657,7 @@ void __init xen_reserve_top(void) · 161 unmodified lines
1657	1657	* Like __va(), but returns address in the kernel mapping (which is
1658	1658	* all we have until the physical memory mapping has been set up.
1659	1659	*/
1660		−static void *__ka(phys_addr_t paddr)
	1660	+static void * __init __ka(phys_addr_t paddr)
1661	1661	{
1662	1662	#ifdef CONFIG_X86_64
1663	1663	return (void *)(paddr + __START_KERNEL_map);
@@ -1667,7 +1667,7 @@ static void *__ka(phys_addr_t paddr) · 3 unmodified lines
1667	1667	}
1668	1668
1669	1669	/* Convert a machine address to physical address */
1670		−static unsigned long m2p(phys_addr_t maddr)
	1670	+static unsigned long __init m2p(phys_addr_t maddr)
1671	1671	{
1672	1672	phys_addr_t paddr;
1673	1673
@@ -1678,13 +1678,14 @@ static unsigned long m2p(phys_addr_t maddr) · 4 unmodified lines
1678	1678	}
1679	1679
1680	1680	/* Convert a machine address to kernel virtual */
1681		−static void *m2v(phys_addr_t maddr)
	1681	+static void * __init m2v(phys_addr_t maddr)
1682	1682	{
1683	1683	return __ka(m2p(maddr));
1684	1684	}
1685	1685
1686	1686	/* Set the page permissions on an identity-mapped pages */
1687		−static void set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags)
	1687	+static void __init set_page_prot_flags(void *addr, pgprot_t prot,
	1688	+ unsigned long flags)
1688	1689	{
1689	1690	unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1690	1691	pte_t pte = pfn_pte(pfn, prot);
@@ -1696,7 +1697,7 @@ static void set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags) · 5 unmodified lines
1696	1697	if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags))
1697	1698	BUG();
1698	1699	}
1699		−static void set_page_prot(void *addr, pgprot_t prot)
	1700	+static void __init set_page_prot(void *addr, pgprot_t prot)
1700	1701	{
1701	1702	return set_page_prot_flags(addr, prot, UVMF_NONE);
1702	1703	}
@@ -1733,10 +1734,8 @@ static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn) · 30 unmodified lines
1733	1734	for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1734	1735	pte_t pte;
1735	1736
1736		−#ifdef CONFIG_X86_32
1737	1737	if (pfn > max_pfn_mapped)
1738	1738	max_pfn_mapped = pfn;
1739		−#endif
1740	1739
1741	1740	if (!pte_none(pte_page[pteidx]))
1742	1741	continue;
@@ -1769,7 +1768,7 @@ void __init xen_setup_machphys_mapping(void) · 26 unmodified lines
1769	1768	}
1770	1769
1771	1770	#ifdef CONFIG_X86_64
1772		−static void convert_pfn_mfn(void *v)
	1771	+static void __init convert_pfn_mfn(void *v)
1773	1772	{
1774	1773	pte_t *pte = v;
1775	1774	int i;

arch/x86/xen/p2m.c+11−256

@@ -84,8 +84,6 @@ · 83 unmodified lines
84	84
85	85	#define PMDS_PER_MID_PAGE (P2M_MID_PER_PAGE / PTRS_PER_PTE)
86	86
87		−static void __init m2p_override_init(void);
88		−
89	87	unsigned long *xen_p2m_addr __read_mostly;
90	88	EXPORT_SYMBOL_GPL(xen_p2m_addr);
91	89	unsigned long xen_p2m_size __read_mostly;
@@ -402,8 +400,6 @@ void __init xen_vmalloc_p2m_tree(void) · 310 unmodified lines
402	400	xen_p2m_size = xen_max_p2m_pfn;
403	401
404	402	xen_inv_extra_mem();
405		−
406		− m2p_override_init();
407	403	}
408	404
409	405	unsigned long get_phys_to_machine(unsigned long pfn)
@@ -652,100 +648,21 @@ bool set_phys_to_machine(unsigned long pfn, unsigned long mfn) · 242 unmodified lines
652	648	return true;
653	649	}
654	650
655		−#define M2P_OVERRIDE_HASH_SHIFT 10
656		−#define M2P_OVERRIDE_HASH (1 << M2P_OVERRIDE_HASH_SHIFT)
657		−
658		−static struct list_head *m2p_overrides;
659		−static DEFINE_SPINLOCK(m2p_override_lock);
660		−
661		−static void __init m2p_override_init(void)
662		−{
663		− unsigned i;
664		−
665		− m2p_overrides = alloc_bootmem_align(
666		− sizeof(m2p_overrides) M2P_OVERRIDE_HASH,
667		− sizeof(unsigned long));
668		−
669		− for (i = 0; i < M2P_OVERRIDE_HASH; i++)
670		− INIT_LIST_HEAD(&m2p_overrides[i]);
671		−}
672		−
673		−static unsigned long mfn_hash(unsigned long mfn)
674		−{
675		− return hash_long(mfn, M2P_OVERRIDE_HASH_SHIFT);
676		−}
677		−
678		−/* Add an MFN override for a particular page */
679		−static int m2p_add_override(unsigned long mfn, struct page *page,
680		− struct gnttab_map_grant_ref *kmap_op)
681		−{
682		− unsigned long flags;
683		− unsigned long pfn;
684		− unsigned long uninitialized_var(address);
685		− unsigned level;
686		− pte_t *ptep = NULL;
687		−
688		− pfn = page_to_pfn(page);
689		− if (!PageHighMem(page)) {
690		− address = (unsigned long)__va(pfn << PAGE_SHIFT);
691		− ptep = lookup_address(address, &level);
692		− if (WARN(ptep == NULL \|\| level != PG_LEVEL_4K,
693		− "m2p_add_override: pfn %lx not mapped", pfn))
694		− return -EINVAL;
695		− }
696		−
697		− if (kmap_op != NULL) {
698		− if (!PageHighMem(page)) {
699		− struct multicall_space mcs =
700		− xen_mc_entry(sizeof(*kmap_op));
701		−
702		− MULTI_grant_table_op(mcs.mc,
703		− GNTTABOP_map_grant_ref, kmap_op, 1);
704		−
705		− xen_mc_issue(PARAVIRT_LAZY_MMU);
706		− }
707		− }
708		− spin_lock_irqsave(&m2p_override_lock, flags);
709		− list_add(&page->lru, &m2p_overrides[mfn_hash(mfn)]);
710		− spin_unlock_irqrestore(&m2p_override_lock, flags);
711		−
712		− /* p2m(m2p(mfn)) == mfn: the mfn is already present somewhere in
713		− * this domain. Set the FOREIGN_FRAME_BIT in the p2m for the other
714		− * pfn so that the following mfn_to_pfn(mfn) calls will return the
715		− * pfn from the m2p_override (the backend pfn) instead.
716		− * We need to do this because the pages shared by the frontend
717		− * (xen-blkfront) can be already locked (lock_page, called by
718		− * do_read_cache_page); when the userspace backend tries to use them
719		− * with direct_IO, mfn_to_pfn returns the pfn of the frontend, so
720		− * do_blockdev_direct_IO is going to try to lock the same pages
721		− * again resulting in a deadlock.
722		− * As a side effect get_user_pages_fast might not be safe on the
723		− * frontend pages while they are being shared with the backend,
724		− * because mfn_to_pfn (that ends up being called by GUPF) will
725		− * return the backend pfn rather than the frontend pfn. */
726		− pfn = mfn_to_pfn_no_overrides(mfn);
727		− if (__pfn_to_mfn(pfn) == mfn)
728		− set_phys_to_machine(pfn, FOREIGN_FRAME(mfn));
729		−
730		− return 0;
731		−}
732		−
733	651	int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
734	652	struct gnttab_map_grant_ref *kmap_ops,
735	653	struct page **pages, unsigned int count)
736	654	{
737	655	int i, ret = 0;
738		− bool lazy = false;
739	656	pte_t *pte;
740	657
741	658	if (xen_feature(XENFEAT_auto_translated_physmap))
742	659	return 0;
743	660
744		− if (kmap_ops &&
745		− !in_interrupt() &&
746		− paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
747		− arch_enter_lazy_mmu_mode();
748		− lazy = true;
	661	+ if (kmap_ops) {
	662	+ ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref,
	663	+ kmap_ops, count);
	664	+ if (ret)
	665	+ goto out;
749	666	}
750	667
751	668	for (i = 0; i < count; i++) {
@@ -764,170 +681,28 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, · 12 unmodified lines
764	681	}
765	682	pfn = page_to_pfn(pages[i]);
766	683
767		− WARN_ON(PagePrivate(pages[i]));
768		− SetPagePrivate(pages[i]);
769		− set_page_private(pages[i], mfn);
770		− pages[i]->index = pfn_to_mfn(pfn);
	684	+ WARN(pfn_to_mfn(pfn) != INVALID_P2M_ENTRY, "page must be ballooned");
771	685
772	686	if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) {
773	687	ret = -ENOMEM;
774	688	goto out;
775	689	}
776		−
777		− if (kmap_ops) {
778		− ret = m2p_add_override(mfn, pages[i], &kmap_ops[i]);
779		− if (ret)
780		− goto out;
781		− }
782	690	}
783	691
784	692	out:
785		− if (lazy)
786		− arch_leave_lazy_mmu_mode();
787		−
788	693	return ret;
789	694	}
790	695	EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping);
791	696
792		−static struct page *m2p_find_override(unsigned long mfn)
793		−{
794		− unsigned long flags;
795		− struct list_head *bucket;
796		− struct page p, ret;
797		−
798		− if (unlikely(!m2p_overrides))
799		− return NULL;
800		−
801		− ret = NULL;
802		− bucket = &m2p_overrides[mfn_hash(mfn)];
803		−
804		− spin_lock_irqsave(&m2p_override_lock, flags);
805		−
806		− list_for_each_entry(p, bucket, lru) {
807		− if (page_private(p) == mfn) {
808		− ret = p;
809		− break;
810		− }
811		− }
812		−
813		− spin_unlock_irqrestore(&m2p_override_lock, flags);
814		−
815		− return ret;
816		−}
817		−
818		−static int m2p_remove_override(struct page *page,
819		− struct gnttab_map_grant_ref *kmap_op,
820		− unsigned long mfn)
821		−{
822		− unsigned long flags;
823		− unsigned long pfn;
824		− unsigned long uninitialized_var(address);
825		− unsigned level;
826		− pte_t *ptep = NULL;
827		−
828		− pfn = page_to_pfn(page);
829		−
830		− if (!PageHighMem(page)) {
831		− address = (unsigned long)__va(pfn << PAGE_SHIFT);
832		− ptep = lookup_address(address, &level);
833		−
834		− if (WARN(ptep == NULL \|\| level != PG_LEVEL_4K,
835		− "m2p_remove_override: pfn %lx not mapped", pfn))
836		− return -EINVAL;
837		− }
838		−
839		− spin_lock_irqsave(&m2p_override_lock, flags);
840		− list_del(&page->lru);
841		− spin_unlock_irqrestore(&m2p_override_lock, flags);
842		−
843		− if (kmap_op != NULL) {
844		− if (!PageHighMem(page)) {
845		− struct multicall_space mcs;
846		− struct gnttab_unmap_and_replace *unmap_op;
847		− struct page *scratch_page = get_balloon_scratch_page();
848		− unsigned long scratch_page_address = (unsigned long)
849		− __va(page_to_pfn(scratch_page) << PAGE_SHIFT);
850		−
851		− /*
852		− * It might be that we queued all the m2p grant table
853		− * hypercalls in a multicall, then m2p_remove_override
854		− * get called before the multicall has actually been
855		− * issued. In this case handle is going to -1 because
856		− * it hasn't been modified yet.
857		− */
858		− if (kmap_op->handle == -1)
859		− xen_mc_flush();
860		− /*
861		− * Now if kmap_op->handle is negative it means that the
862		− * hypercall actually returned an error.
863		− */
864		− if (kmap_op->handle == GNTST_general_error) {
865		− pr_warn("m2p_remove_override: pfn %lx mfn %lx, failed to modify kernel mappings",
866		− pfn, mfn);
867		− put_balloon_scratch_page();
868		− return -1;
869		− }
870		−
871		− xen_mc_batch();
872		−
873		− mcs = __xen_mc_entry(
874		− sizeof(struct gnttab_unmap_and_replace));
875		− unmap_op = mcs.args;
876		− unmap_op->host_addr = kmap_op->host_addr;
877		− unmap_op->new_addr = scratch_page_address;
878		− unmap_op->handle = kmap_op->handle;
879		−
880		− MULTI_grant_table_op(mcs.mc,
881		− GNTTABOP_unmap_and_replace, unmap_op, 1);
882		−
883		− mcs = __xen_mc_entry(0);
884		− MULTI_update_va_mapping(mcs.mc, scratch_page_address,
885		− pfn_pte(page_to_pfn(scratch_page),
886		− PAGE_KERNEL_RO), 0);
887		−
888		− xen_mc_issue(PARAVIRT_LAZY_MMU);
889		−
890		− kmap_op->host_addr = 0;
891		− put_balloon_scratch_page();
892		− }
893		− }
894		−
895		− /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present
896		− * somewhere in this domain, even before being added to the
897		− * m2p_override (see comment above in m2p_add_override).
898		− * If there are no other entries in the m2p_override corresponding
899		− * to this mfn, then remove the FOREIGN_FRAME_BIT from the p2m for
900		− * the original pfn (the one shared by the frontend): the backend
901		− * cannot do any IO on this page anymore because it has been
902		− * unshared. Removing the FOREIGN_FRAME_BIT from the p2m entry of
903		− * the original pfn causes mfn_to_pfn(mfn) to return the frontend
904		− * pfn again. */
905		− mfn &= ~FOREIGN_FRAME_BIT;
906		− pfn = mfn_to_pfn_no_overrides(mfn);
907		− if (__pfn_to_mfn(pfn) == FOREIGN_FRAME(mfn) &&
908		− m2p_find_override(mfn) == NULL)
909		− set_phys_to_machine(pfn, mfn);
910		−
911		− return 0;
912		−}
913		−
914	697	int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
915		− struct gnttab_map_grant_ref *kmap_ops,
	698	+ struct gnttab_unmap_grant_ref *kunmap_ops,
916	699	struct page **pages, unsigned int count)
917	700	{
918	701	int i, ret = 0;
919		− bool lazy = false;
920	702
921	703	if (xen_feature(XENFEAT_auto_translated_physmap))
922	704	return 0;
923	705
924		− if (kmap_ops &&
925		− !in_interrupt() &&
926		− paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
927		− arch_enter_lazy_mmu_mode();
928		− lazy = true;
929		− }
930		−
931	706	for (i = 0; i < count; i++) {
932	707	unsigned long mfn = __pfn_to_mfn(page_to_pfn(pages[i]));
933	708	unsigned long pfn = page_to_pfn(pages[i]);
@@ -937,36 +712,16 @@ int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, · 3 unmodified lines
937	712	goto out;
938	713	}
939	714
940		− set_page_private(pages[i], INVALID_P2M_ENTRY);
941		− WARN_ON(!PagePrivate(pages[i]));
942		− ClearPagePrivate(pages[i]);
943		− set_phys_to_machine(pfn, pages[i]->index);
944		−
945		− if (kmap_ops)
946		− ret = m2p_remove_override(pages[i], &kmap_ops[i], mfn);
947		− if (ret)
948		− goto out;
	715	+ set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
949	716	}
950		−
	717	+ if (kunmap_ops)
	718	+ ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref,
	719	+ kunmap_ops, count);
951	720	out:
952		− if (lazy)
953		− arch_leave_lazy_mmu_mode();
954	721	return ret;
955	722	}
956	723	EXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping);
957	724
958		−unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn)
959		−{
960		− struct page *p = m2p_find_override(mfn);
961		− unsigned long ret = pfn;
962		−
963		− if (p)
964		− ret = page_to_pfn(p);
965		−
966		− return ret;
967		−}
968		−EXPORT_SYMBOL_GPL(m2p_find_override_pfn);
969		−
970	725	#ifdef CONFIG_XEN_DEBUG_FS
971	726	#include <linux/debugfs.h>
972	727	#include "debugfs.h"

arch/x86/xen/setup.c+14−23

@@ -32,16 +32,6 @@ · 31 unmodified lines
32	32	#include "p2m.h"
33	33	#include "mmu.h"
34	34
35		−/* These are code, but not functions. Defined in entry.S */
36		−extern const char xen_hypervisor_callback[];
37		−extern const char xen_failsafe_callback[];
38		−#ifdef CONFIG_X86_64
39		−extern asmlinkage void nmi(void);
40		−#endif
41		−extern void xen_sysenter_target(void);
42		−extern void xen_syscall_target(void);
43		−extern void xen_syscall32_target(void);
44		−
45	35	/* Amount of extra memory space we add to the e820 ranges */
46	36	struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata;
47	37
@@ -74,7 +64,7 @@ static unsigned long xen_remap_mfn __initdata = INVALID_P2M_ENTRY; · 26 unmodified lines
74	64	*/
75	65	#define EXTRA_MEM_RATIO (10)
76	66
77		−static void __init xen_add_extra_mem(u64 start, u64 size)
	67	+static void __init xen_add_extra_mem(phys_addr_t start, phys_addr_t size)
78	68	{
79	69	int i;
80	70
@@ -97,10 +87,10 @@ static void __init xen_add_extra_mem(u64 start, u64 size) · 16 unmodified lines
97	87	memblock_reserve(start, size);
98	88	}
99	89
100		−static void __init xen_del_extra_mem(u64 start, u64 size)
	90	+static void __init xen_del_extra_mem(phys_addr_t start, phys_addr_t size)
101	91	{
102	92	int i;
103		− u64 start_r, size_r;
	93	+ phys_addr_t start_r, size_r;
104	94
105	95	for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) {
106	96	start_r = xen_extra_mem[i].start;
@@ -267,7 +257,7 @@ static void __init xen_set_identity_and_release_chunk(unsigned long start_pfn, · 160 unmodified lines
267	257	static void __init xen_update_mem_tables(unsigned long pfn, unsigned long mfn)
268	258	{
269	259	struct mmu_update update = {
270		− .ptr = ((unsigned long long)mfn << PAGE_SHIFT) \| MMU_MACHPHYS_UPDATE,
	260	+ .ptr = ((uint64_t)mfn << PAGE_SHIFT) \| MMU_MACHPHYS_UPDATE,
271	261	.val = pfn
272	262	};
273	263
@@ -545,20 +535,21 @@ static unsigned long __init xen_get_max_pages(void) · 271 unmodified lines
545	535	return min(max_pages, MAX_DOMAIN_PAGES);
546	536	}
547	537
548		−static void xen_align_and_add_e820_region(u64 start, u64 size, int type)
	538	+static void __init xen_align_and_add_e820_region(phys_addr_t start,
	539	+ phys_addr_t size, int type)
549	540	{
550		− u64 end = start + size;
	541	+ phys_addr_t end = start + size;
551	542
552	543	/* Align RAM regions to page boundaries. */
553	544	if (type == E820_RAM) {
554	545	start = PAGE_ALIGN(start);
555		− end &= ~((u64)PAGE_SIZE - 1);
	546	+ end &= ~((phys_addr_t)PAGE_SIZE - 1);
556	547	}
557	548
558	549	e820_add_region(start, end - start, type);
559	550	}
560	551
561		−void xen_ignore_unusable(struct e820entry *list, size_t map_size)
	552	+static void __init xen_ignore_unusable(struct e820entry *list, size_t map_size)
562	553	{
563	554	struct e820entry *entry;
564	555	unsigned int i;
@@ -577,7 +568,7 @@ char * __init xen_memory_setup(void) · 12 unmodified lines
577	568	static struct e820entry map[E820MAX] __initdata;
578	569
579	570	unsigned long max_pfn = xen_start_info->nr_pages;
580		− unsigned long long mem_end;
	571	+ phys_addr_t mem_end;
581	572	int rc;
582	573	struct xen_memory_map memmap;
583	574	unsigned long max_pages;
@@ -652,16 +643,16 @@ char * __init xen_memory_setup(void) · 68 unmodified lines
652	643	extra_pages);
653	644	i = 0;
654	645	while (i < memmap.nr_entries) {
655		− u64 addr = map[i].addr;
656		− u64 size = map[i].size;
	646	+ phys_addr_t addr = map[i].addr;
	647	+ phys_addr_t size = map[i].size;
657	648	u32 type = map[i].type;
658	649
659	650	if (type == E820_RAM) {
660	651	if (addr < mem_end) {
661	652	size = min(size, mem_end - addr);
662	653	} else if (extra_pages) {
663		− size = min(size, (u64)extra_pages * PAGE_SIZE);
664		− extra_pages -= size / PAGE_SIZE;
	654	+ size = min(size, PFN_PHYS(extra_pages));
	655	+ extra_pages -= PFN_DOWN(size);
665	656	xen_add_extra_mem(addr, size);
666	657	xen_max_p2m_pfn = PFN_DOWN(addr + size);
667	658	} else

arch/x86/xen/smp.c+1−1

@@ -507,7 +507,7 @@ static int xen_cpu_disable(void) · 506 unmodified lines
507	507	static void xen_cpu_die(unsigned int cpu)
508	508	{
509	509	while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
510		− current->state = TASK_UNINTERRUPTIBLE;
	510	+ __set_current_state(TASK_UNINTERRUPTIBLE);
511	511	schedule_timeout(HZ/10);
512	512	}
513	513

arch/x86/xen/time.c+4−0

@@ -479,6 +479,10 @@ static void __init xen_time_init(void) · 478 unmodified lines
479	479	int cpu = smp_processor_id();
480	480	struct timespec tp;
481	481
	482	+ /* As Dom0 is never moved, no penalty on using TSC there */
	483	+ if (xen_initial_domain())
	484	+ xen_clocksource.rating = 275;
	485	+
482	486	clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
483	487
484	488	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {

arch/x86/xen/xen-ops.h+6−0

@@ -10,6 +10,12 @@ · 9 unmodified lines
10	10	extern const char xen_hypervisor_callback[];
11	11	extern const char xen_failsafe_callback[];
12	12
	13	+void xen_sysenter_target(void);
	14	+#ifdef CONFIG_X86_64
	15	+void xen_syscall_target(void);
	16	+void xen_syscall32_target(void);
	17	+#endif
	18	+
13	19	extern void *xen_initial_gdt;
14	20
15	21	struct trap_info;

drivers/block/xen-blkback/blkback.c+123−54

@@ -47,6 +47,7 @@ · 46 unmodified lines
47	47	#include <asm/xen/hypervisor.h>
48	48	#include <asm/xen/hypercall.h>
49	49	#include <xen/balloon.h>
	50	+#include <xen/grant_table.h>
50	51	#include "common.h"
51	52
52	53	/*
@@ -100,7 +101,7 @@ module_param(log_stats, int, 0644); · 47 unmodified lines
100	101
101	102	#define BLKBACK_INVALID_HANDLE (~0)
102	103
103		−/* Number of free pages to remove on each call to free_xenballooned_pages */
	104	+/* Number of free pages to remove on each call to gnttab_free_pages */
104	105	#define NUM_BATCH_FREE_PAGES 10
105	106
106	107	static inline int get_free_page(struct xen_blkif blkif, struct page *page)
@@ -111,7 +112,7 @@ static inline int get_free_page(struct xen_blkif blkif, struct page *page) · 4 unmodified lines
111	112	if (list_empty(&blkif->free_pages)) {
112	113	BUG_ON(blkif->free_pages_num != 0);
113	114	spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
114		− return alloc_xenballooned_pages(1, page, false);
	115	+ return gnttab_alloc_pages(1, page);
115	116	}
116	117	BUG_ON(blkif->free_pages_num == 0);
117	118	page[0] = list_first_entry(&blkif->free_pages, struct page, lru);
@@ -151,14 +152,14 @@ static inline void shrink_free_pagepool(struct xen_blkif *blkif, int num) · 33 unmodified lines
151	152	blkif->free_pages_num--;
152	153	if (++num_pages == NUM_BATCH_FREE_PAGES) {
153	154	spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
154		− free_xenballooned_pages(num_pages, page);
	155	+ gnttab_free_pages(num_pages, page);
155	156	spin_lock_irqsave(&blkif->free_pages_lock, flags);
156	157	num_pages = 0;
157	158	}
158	159	}
159	160	spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
160	161	if (num_pages != 0)
161		− free_xenballooned_pages(num_pages, page);
	162	+ gnttab_free_pages(num_pages, page);
162	163	}
163	164
164	165	#define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page)))
@@ -262,6 +263,17 @@ static void put_persistent_gnt(struct xen_blkif *blkif, · 97 unmodified lines
262	263	atomic_dec(&blkif->persistent_gnt_in_use);
263	264	}
264	265
	266	+static void free_persistent_gnts_unmap_callback(int result,
	267	+ struct gntab_unmap_queue_data *data)
	268	+{
	269	+ struct completion *c = data->data;
	270	+
	271	+ /* BUG_ON used to reproduce existing behaviour,
	272	+ but is this the best way to deal with this? */
	273	+ BUG_ON(result);
	274	+ complete(c);
	275	+}
	276	+
265	277	static void free_persistent_gnts(struct xen_blkif blkif, struct rb_root root,
266	278	unsigned int num)
267	279	{
@@ -269,8 +281,17 @@ static void free_persistent_gnts(struct xen_blkif blkif, struct rb_root root, · 1 unmodified line
269	281	struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
270	282	struct persistent_gnt *persistent_gnt;
271	283	struct rb_node *n;
272		− int ret = 0;
273	284	int segs_to_unmap = 0;
	285	+ struct gntab_unmap_queue_data unmap_data;
	286	+ struct completion unmap_completion;
	287	+
	288	+ init_completion(&unmap_completion);
	289	+
	290	+ unmap_data.data = &unmap_completion;
	291	+ unmap_data.done = &free_persistent_gnts_unmap_callback;
	292	+ unmap_data.pages = pages;
	293	+ unmap_data.unmap_ops = unmap;
	294	+ unmap_data.kunmap_ops = NULL;
274	295
275	296	foreach_grant_safe(persistent_gnt, n, root, node) {
276	297	BUG_ON(persistent_gnt->handle ==
@@ -285,9 +306,11 @@ static void free_persistent_gnts(struct xen_blkif blkif, struct rb_root root, · 8 unmodified lines
285	306
286	307	if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST \|\|
287	308	!rb_next(&persistent_gnt->node)) {
288		− ret = gnttab_unmap_refs(unmap, NULL, pages,
289		− segs_to_unmap);
290		− BUG_ON(ret);
	309	+
	310	+ unmap_data.count = segs_to_unmap;
	311	+ gnttab_unmap_refs_async(&unmap_data);
	312	+ wait_for_completion(&unmap_completion);
	313	+
291	314	put_free_pages(blkif, pages, segs_to_unmap);
292	315	segs_to_unmap = 0;
293	316	}
@@ -653,18 +676,14 @@ void xen_blkbk_free_caches(struct xen_blkif *blkif) · 359 unmodified lines
653	676	shrink_free_pagepool(blkif, 0 /* All */);
654	677	}
655	678
656		−/*
657		− * Unmap the grant references, and also remove the M2P over-rides
658		− * used in the 'pending_req'.
659		− */
660		−static void xen_blkbk_unmap(struct xen_blkif *blkif,
661		− struct grant_page *pages[],
662		− int num)
	679	+static unsigned int xen_blkbk_unmap_prepare(
	680	+ struct xen_blkif *blkif,
	681	+ struct grant_page **pages,
	682	+ unsigned int num,
	683	+ struct gnttab_unmap_grant_ref *unmap_ops,
	684	+ struct page **unmap_pages)
663	685	{
664		− struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
665		− struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
666	686	unsigned int i, invcount = 0;
667		− int ret;
668	687
669	688	for (i = 0; i < num; i++) {
670	689	if (pages[i]->persistent_gnt != NULL) {
@@ -674,21 +693,95 @@ static void xen_blkbk_unmap(struct xen_blkif *blkif, · 3 unmodified lines
674	693	if (pages[i]->handle == BLKBACK_INVALID_HANDLE)
675	694	continue;
676	695	unmap_pages[invcount] = pages[i]->page;
677		− gnttab_set_unmap_op(&unmap[invcount], vaddr(pages[i]->page),
	696	+ gnttab_set_unmap_op(&unmap_ops[invcount], vaddr(pages[i]->page),
678	697	GNTMAP_host_map, pages[i]->handle);
679	698	pages[i]->handle = BLKBACK_INVALID_HANDLE;
680		− if (++invcount == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
681		− ret = gnttab_unmap_refs(unmap, NULL, unmap_pages,
682		− invcount);
	699	+ invcount++;
	700	+ }
	701	+
	702	+ return invcount;
	703	+}
	704	+
	705	+static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data)
	706	+{
	707	+ struct pending_req* pending_req = (struct pending_req*) (data->data);
	708	+ struct xen_blkif *blkif = pending_req->blkif;
	709	+
	710	+ /* BUG_ON used to reproduce existing behaviour,
	711	+ but is this the best way to deal with this? */
	712	+ BUG_ON(result);
	713	+
	714	+ put_free_pages(blkif, data->pages, data->count);
	715	+ make_response(blkif, pending_req->id,
	716	+ pending_req->operation, pending_req->status);
	717	+ free_req(blkif, pending_req);
	718	+ /*
	719	+ * Make sure the request is freed before releasing blkif,
	720	+ * or there could be a race between free_req and the
	721	+ * cleanup done in xen_blkif_free during shutdown.
	722	+ *
	723	+ * NB: The fact that we might try to wake up pending_free_wq
	724	+ * before drain_complete (in case there's a drain going on)
	725	+ * it's not a problem with our current implementation
	726	+ * because we can assure there's no thread waiting on
	727	+ * pending_free_wq if there's a drain going on, but it has
	728	+ * to be taken into account if the current model is changed.
	729	+ */
	730	+ if (atomic_dec_and_test(&blkif->inflight) && atomic_read(&blkif->drain)) {
	731	+ complete(&blkif->drain_complete);
	732	+ }
	733	+ xen_blkif_put(blkif);
	734	+}
	735	+
	736	+static void xen_blkbk_unmap_and_respond(struct pending_req *req)
	737	+{
	738	+ struct gntab_unmap_queue_data* work = &req->gnttab_unmap_data;
	739	+ struct xen_blkif *blkif = req->blkif;
	740	+ struct grant_page **pages = req->segments;
	741	+ unsigned int invcount;
	742	+
	743	+ invcount = xen_blkbk_unmap_prepare(blkif, pages, req->nr_pages,
	744	+ req->unmap, req->unmap_pages);
	745	+
	746	+ work->data = req;
	747	+ work->done = xen_blkbk_unmap_and_respond_callback;
	748	+ work->unmap_ops = req->unmap;
	749	+ work->kunmap_ops = NULL;
	750	+ work->pages = req->unmap_pages;
	751	+ work->count = invcount;
	752	+
	753	+ gnttab_unmap_refs_async(&req->gnttab_unmap_data);
	754	+}
	755	+
	756	+
	757	+/*
	758	+ * Unmap the grant references.
	759	+ *
	760	+ * This could accumulate ops up to the batch size to reduce the number
	761	+ * of hypercalls, but since this is only used in error paths there's
	762	+ * no real need.
	763	+ */
	764	+static void xen_blkbk_unmap(struct xen_blkif *blkif,
	765	+ struct grant_page *pages[],
	766	+ int num)
	767	+{
	768	+ struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
	769	+ struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
	770	+ unsigned int invcount = 0;
	771	+ int ret;
	772	+
	773	+ while (num) {
	774	+ unsigned int batch = min(num, BLKIF_MAX_SEGMENTS_PER_REQUEST);
	775	+
	776	+ invcount = xen_blkbk_unmap_prepare(blkif, pages, batch,
	777	+ unmap, unmap_pages);
	778	+ if (invcount) {
	779	+ ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
683	780	BUG_ON(ret);
684	781	put_free_pages(blkif, unmap_pages, invcount);
685		− invcount = 0;
686	782	}
687		− }
688		− if (invcount) {
689		− ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
690		− BUG_ON(ret);
691		− put_free_pages(blkif, unmap_pages, invcount);
	783	+ pages += batch;
	784	+ num -= batch;
692	785	}
693	786	}
694	787
@@ -982,32 +1075,8 @@ static void __end_block_io_op(struct pending_req *pending_req, int error) · 287 unmodified lines
982	1075	* the grant references associated with 'request' and provide
983	1076	* the proper response on the ring.
984	1077	*/
985		− if (atomic_dec_and_test(&pending_req->pendcnt)) {
986		− struct xen_blkif *blkif = pending_req->blkif;
987		−
988		− xen_blkbk_unmap(blkif,
989		− pending_req->segments,
990		− pending_req->nr_pages);
991		− make_response(blkif, pending_req->id,
992		− pending_req->operation, pending_req->status);
993		− free_req(blkif, pending_req);
994		− /*
995		− * Make sure the request is freed before releasing blkif,
996		− * or there could be a race between free_req and the
997		− * cleanup done in xen_blkif_free during shutdown.
998		− *
999		− * NB: The fact that we might try to wake up pending_free_wq
1000		− * before drain_complete (in case there's a drain going on)
1001		− * it's not a problem with our current implementation
1002		− * because we can assure there's no thread waiting on
1003		− * pending_free_wq if there's a drain going on, but it has
1004		− * to be taken into account if the current model is changed.
1005		− */
1006		− if (atomic_dec_and_test(&blkif->inflight) && atomic_read(&blkif->drain)) {
1007		− complete(&blkif->drain_complete);
1008		− }
1009		− xen_blkif_put(blkif);
1010		− }
	1078	+ if (atomic_dec_and_test(&pending_req->pendcnt))
	1079	+ xen_blkbk_unmap_and_respond(pending_req);
1011	1080	}
1012	1081
1013	1082	/*

drivers/block/xen-blkback/common.h+3−0

@@ -350,6 +350,9 @@ struct pending_req { · 349 unmodified lines
350	350	struct grant_page *indirect_pages[MAX_INDIRECT_PAGES];
351	351	struct seg_buf seg[MAX_INDIRECT_SEGMENTS];
352	352	struct bio *biolist[MAX_INDIRECT_SEGMENTS];
	353	+ struct gnttab_unmap_grant_ref unmap[MAX_INDIRECT_SEGMENTS];
	354	+ struct page *unmap_pages[MAX_INDIRECT_SEGMENTS];
	355	+ struct gntab_unmap_queue_data gnttab_unmap_data;
353	356	};
354	357
355	358

drivers/net/xen-netback/interface.c+3−4

@@ -483,9 +483,8 @@ int xenvif_init_queue(struct xenvif_queue *queue) · 482 unmodified lines
483	483	* better enable it. The long term solution would be to use just a
484	484	* bunch of valid page descriptors, without dependency on ballooning
485	485	*/
486		− err = alloc_xenballooned_pages(MAX_PENDING_REQS,
487		− queue->mmap_pages,
488		− false);
	486	+ err = gnttab_alloc_pages(MAX_PENDING_REQS,
	487	+ queue->mmap_pages);
489	488	if (err) {
490	489	netdev_err(queue->vif->dev, "Could not reserve mmap_pages\n");
491	490	return -ENOMEM;
@@ -664,7 +663,7 @@ void xenvif_disconnect(struct xenvif *vif) · 172 unmodified lines
664	663	*/
665	664	void xenvif_deinit_queue(struct xenvif_queue *queue)
666	665	{
667		− free_xenballooned_pages(MAX_PENDING_REQS, queue->mmap_pages);
	666	+ gnttab_free_pages(MAX_PENDING_REQS, queue->mmap_pages);
668	667	}
669	668
670	669	void xenvif_free(struct xenvif *vif)

drivers/net/xen-netback/netback.c+9−97

@@ -314,9 +314,7 @@ static struct xenvif_rx_meta get_next_rx_buffer(struct xenvif_queue queue, · 313 unmodified lines
314	314	static void xenvif_gop_frag_copy(struct xenvif_queue queue, struct sk_buff skb,
315	315	struct netrx_pending_operations *npo,
316	316	struct page *page, unsigned long size,
317		− unsigned long offset, int *head,
318		− struct xenvif_queue *foreign_queue,
319		− grant_ref_t foreign_gref)
	317	+ unsigned long offset, int *head)
320	318	{
321	319	struct gnttab_copy *copy_gop;
322	320	struct xenvif_rx_meta *meta;
@@ -333,6 +331,8 @@ static void xenvif_gop_frag_copy(struct xenvif_queue queue, struct sk_buff skb · 10 unmodified lines
333	331	offset &= ~PAGE_MASK;
334	332
335	333	while (size > 0) {
	334	+ struct xen_page_foreign *foreign;
	335	+
336	336	BUG_ON(offset >= PAGE_SIZE);
337	337	BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET);
338	338
@@ -361,9 +361,10 @@ static void xenvif_gop_frag_copy(struct xenvif_queue queue, struct sk_buff skb · 22 unmodified lines
361	361	copy_gop->flags = GNTCOPY_dest_gref;
362	362	copy_gop->len = bytes;
363	363
364		− if (foreign_queue) {
365		− copy_gop->source.domid = foreign_queue->vif->domid;
366		− copy_gop->source.u.ref = foreign_gref;
	364	+ foreign = xen_page_foreign(page);
	365	+ if (foreign) {
	366	+ copy_gop->source.domid = foreign->domid;
	367	+ copy_gop->source.u.ref = foreign->gref;
367	368	copy_gop->flags \|= GNTCOPY_source_gref;
368	369	} else {
369	370	copy_gop->source.domid = DOMID_SELF;
@@ -405,35 +406,6 @@ static void xenvif_gop_frag_copy(struct xenvif_queue queue, struct sk_buff skb · 35 unmodified lines
405	406	}
406	407	}
407	408
408		−/*
409		− * Find the grant ref for a given frag in a chain of struct ubuf_info's
410		− * skb: the skb itself
411		− * i: the frag's number
412		− * ubuf: a pointer to an element in the chain. It should not be NULL
413		− *
414		− * Returns a pointer to the element in the chain where the page were found. If
415		− * not found, returns NULL.
416		− * See the definition of callback_struct in common.h for more details about
417		− * the chain.
418		− */
419		−static const struct ubuf_info xenvif_find_gref(const struct sk_buff const skb,
420		− const int i,
421		− const struct ubuf_info *ubuf)
422		−{
423		− struct xenvif_queue *foreign_queue = ubuf_to_queue(ubuf);
424		−
425		− do {
426		− u16 pending_idx = ubuf->desc;
427		−
428		− if (skb_shinfo(skb)->frags[i].page.p ==
429		− foreign_queue->mmap_pages[pending_idx])
430		− break;
431		− ubuf = (struct ubuf_info *) ubuf->ctx;
432		− } while (ubuf);
433		−
434		− return ubuf;
435		−}
436		−
437	409	/*
438	410	* Prepare an SKB to be transmitted to the frontend.
439	411	*
@@ -459,8 +431,6 @@ static int xenvif_gop_skb(struct sk_buff *skb, · 19 unmodified lines
459	431	int head = 1;
460	432	int old_meta_prod;
461	433	int gso_type;
462		− const struct ubuf_info *ubuf = skb_shinfo(skb)->destructor_arg;
463		− const struct ubuf_info *const head_ubuf = ubuf;
464	434
465	435	old_meta_prod = npo->meta_prod;
466	436
@@ -507,68 +477,16 @@ static int xenvif_gop_skb(struct sk_buff *skb, · 40 unmodified lines
507	477	len = skb_tail_pointer(skb) - data;
508	478
509	479	xenvif_gop_frag_copy(queue, skb, npo,
510		− virt_to_page(data), len, offset, &head,
511		− NULL,
512		− 0);
	480	+ virt_to_page(data), len, offset, &head);
513	481	data += len;
514	482	}
515	483
516	484	for (i = 0; i < nr_frags; i++) {
517		− /* This variable also signals whether foreign_gref has a real
518		− * value or not.
519		− */
520		− struct xenvif_queue *foreign_queue = NULL;
521		− grant_ref_t foreign_gref;
522		−
523		− if ((skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) &&
524		− (ubuf->callback == &xenvif_zerocopy_callback)) {
525		− const struct ubuf_info *const startpoint = ubuf;
526		−
527		− /* Ideally ubuf points to the chain element which
528		− * belongs to this frag. Or if frags were removed from
529		− * the beginning, then shortly before it.
530		− */
531		− ubuf = xenvif_find_gref(skb, i, ubuf);
532		−
533		− /* Try again from the beginning of the list, if we
534		− * haven't tried from there. This only makes sense in
535		− * the unlikely event of reordering the original frags.
536		− * For injected local pages it's an unnecessary second
537		− * run.
538		− */
539		− if (unlikely(!ubuf) && startpoint != head_ubuf)
540		− ubuf = xenvif_find_gref(skb, i, head_ubuf);
541		−
542		− if (likely(ubuf)) {
543		− u16 pending_idx = ubuf->desc;
544		−
545		− foreign_queue = ubuf_to_queue(ubuf);
546		− foreign_gref =
547		− foreign_queue->pending_tx_info[pending_idx].req.gref;
548		− /* Just a safety measure. If this was the last
549		− * element on the list, the for loop will
550		− * iterate again if a local page were added to
551		− * the end. Using head_ubuf here prevents the
552		− * second search on the chain. Or the original
553		− * frags changed order, but that's less likely.
554		− * In any way, ubuf shouldn't be NULL.
555		− */
556		− ubuf = ubuf->ctx ?
557		− (struct ubuf_info *) ubuf->ctx :
558		− head_ubuf;
559		− } else
560		− /* This frag was a local page, added to the
561		− * array after the skb left netback.
562		− */
563		− ubuf = head_ubuf;
564		− }
565	485	xenvif_gop_frag_copy(queue, skb, npo,
566	486	skb_frag_page(&skb_shinfo(skb)->frags[i]),
567	487	skb_frag_size(&skb_shinfo(skb)->frags[i]),
568	488	skb_shinfo(skb)->frags[i].page_offset,
569		− &head,
570		− foreign_queue,
571		− foreign_queue ? foreign_gref : UINT_MAX);
	489	+ &head);
572	490	}
573	491
574	492	return npo->meta_prod - old_meta_prod;
@@ -1241,12 +1159,6 @@ static void xenvif_fill_frags(struct xenvif_queue queue, struct sk_buff skb) · 666 unmodified lines
1241	1159	/* Take an extra reference to offset network stack's put_page */
1242	1160	get_page(queue->mmap_pages[pending_idx]);
1243	1161	}
1244		− /* FIXME: __skb_fill_page_desc set this to true because page->pfmemalloc
1245		− * overlaps with "index", and "mapping" is not set. I think mapping
1246		− * should be set. If delivered to local stack, it would drop this
1247		− * skb in sk_filter unless the socket has the right to use it.
1248		− */
1249		− skb->pfmemalloc = false;
1250	1162	}
1251	1163
1252	1164	static int xenvif_get_extras(struct xenvif_queue *queue,

drivers/xen/balloon.c+2−84

@@ -92,7 +92,6 @@ EXPORT_SYMBOL_GPL(balloon_stats); · 91 unmodified lines
92	92
93	93	/* We increase/decrease in batches which fit in a page */
94	94	static xen_pfn_t frame_list[PAGE_SIZE / sizeof(unsigned long)];
95		−static DEFINE_PER_CPU(struct page *, balloon_scratch_page);
96	95
97	96
98	97	/* List of ballooned pages, threaded through the mem_map array. */
@@ -423,22 +422,12 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp) · 324 unmodified lines
423	422	page = pfn_to_page(pfn);
424	423
425	424	#ifdef CONFIG_XEN_HAVE_PVMMU
426		− /*
427		− * Ballooned out frames are effectively replaced with
428		− * a scratch frame. Ensure direct mappings and the
429		− * p2m are consistent.
430		− */
431	425	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
432	426	if (!PageHighMem(page)) {
433		− struct page *scratch_page = get_balloon_scratch_page();
434		−
435	427	ret = HYPERVISOR_update_va_mapping(
436	428	(unsigned long)__va(pfn << PAGE_SHIFT),
437		− pfn_pte(page_to_pfn(scratch_page),
438		− PAGE_KERNEL_RO), 0);
	429	+ __pte_ma(0), 0);
439	430	BUG_ON(ret);
440		−
441		− put_balloon_scratch_page();
442	431	}
443	432	__set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
444	433	}
@@ -500,18 +489,6 @@ static void balloon_process(struct work_struct *work) · 55 unmodified lines
500	489	mutex_unlock(&balloon_mutex);
501	490	}
502	491
503		−struct page *get_balloon_scratch_page(void)
504		−{
505		− struct page *ret = get_cpu_var(balloon_scratch_page);
506		− BUG_ON(ret == NULL);
507		− return ret;
508		−}
509		−
510		−void put_balloon_scratch_page(void)
511		−{
512		− put_cpu_var(balloon_scratch_page);
513		−}
514		−
515	492	/* Resets the Xen limit, sets new target, and kicks off processing. */
516	493	void balloon_set_new_target(unsigned long target)
517	494	{
@@ -605,61 +582,13 @@ static void __init balloon_add_region(unsigned long start_pfn, · 87 unmodified lines
605	582	}
606	583	}
607	584
608		−static int alloc_balloon_scratch_page(int cpu)
609		−{
610		− if (per_cpu(balloon_scratch_page, cpu) != NULL)
611		− return 0;
612		−
613		− per_cpu(balloon_scratch_page, cpu) = alloc_page(GFP_KERNEL);
614		− if (per_cpu(balloon_scratch_page, cpu) == NULL) {
615		− pr_warn("Failed to allocate balloon_scratch_page for cpu %d\n", cpu);
616		− return -ENOMEM;
617		− }
618		−
619		− return 0;
620		−}
621		−
622		−
623		−static int balloon_cpu_notify(struct notifier_block *self,
624		− unsigned long action, void *hcpu)
625		−{
626		− int cpu = (long)hcpu;
627		− switch (action) {
628		− case CPU_UP_PREPARE:
629		− if (alloc_balloon_scratch_page(cpu))
630		− return NOTIFY_BAD;
631		− break;
632		− default:
633		− break;
634		− }
635		− return NOTIFY_OK;
636		−}
637		−
638		−static struct notifier_block balloon_cpu_notifier = {
639		− .notifier_call = balloon_cpu_notify,
640		−};
641		−
642	585	static int __init balloon_init(void)
643	586	{
644		− int i, cpu;
	587	+ int i;
645	588
646	589	if (!xen_domain())
647	590	return -ENODEV;
648	591
649		− if (!xen_feature(XENFEAT_auto_translated_physmap)) {
650		− register_cpu_notifier(&balloon_cpu_notifier);
651		−
652		− get_online_cpus();
653		− for_each_online_cpu(cpu) {
654		− if (alloc_balloon_scratch_page(cpu)) {
655		− put_online_cpus();
656		− unregister_cpu_notifier(&balloon_cpu_notifier);
657		− return -ENOMEM;
658		− }
659		− }
660		− put_online_cpus();
661		− }
662		−
663	592	pr_info("Initialising balloon driver\n");
664	593
665	594	balloon_stats.current_pages = xen_pv_domain()
@@ -696,15 +625,4 @@ static int __init balloon_init(void) · 30 unmodified lines
696	625
697	626	subsys_initcall(balloon_init);
698	627
699		−static int __init balloon_clear(void)
700		−{
701		− int cpu;
702		−
703		− for_each_possible_cpu(cpu)
704		− per_cpu(balloon_scratch_page, cpu) = NULL;
705		−
706		− return 0;
707		−}
708		−early_initcall(balloon_clear);
709		−
710	628	MODULE_LICENSE("GPL");

drivers/xen/gntdev.c+111−32

@@ -67,7 +67,7 @@ struct gntdev_priv { · 66 unmodified lines
67	67	* Only populated if populate_freeable_maps == 1 */
68	68	struct list_head freeable_maps;
69	69	/* lock protects maps and freeable_maps */
70		− spinlock_t lock;
	70	+ struct mutex lock;
71	71	struct mm_struct *mm;
72	72	struct mmu_notifier mn;
73	73	};
@@ -91,7 +91,9 @@ struct grant_map { · 17 unmodified lines
91	91	struct gnttab_map_grant_ref *map_ops;
92	92	struct gnttab_unmap_grant_ref *unmap_ops;
93	93	struct gnttab_map_grant_ref *kmap_ops;
	94	+ struct gnttab_unmap_grant_ref *kunmap_ops;
94	95	struct page **pages;
	96	+ unsigned long pages_vm_start;
95	97	};
96	98
97	99	static int unmap_grant_pages(struct grant_map *map, int offset, int pages);
@@ -118,12 +120,13 @@ static void gntdev_free_map(struct grant_map *map) · 20 unmodified lines
118	120	return;
119	121
120	122	if (map->pages)
121		− free_xenballooned_pages(map->count, map->pages);
	123	+ gnttab_free_pages(map->count, map->pages);
122	124	kfree(map->pages);
123	125	kfree(map->grants);
124	126	kfree(map->map_ops);
125	127	kfree(map->unmap_ops);
126	128	kfree(map->kmap_ops);
	129	+ kfree(map->kunmap_ops);
127	130	kfree(map);
128	131	}
129	132
@@ -140,21 +143,24 @@ static struct grant_map gntdev_alloc_map(struct gntdev_priv priv, int count) · 10 unmodified lines
140	143	add->map_ops = kcalloc(count, sizeof(add->map_ops[0]), GFP_KERNEL);
141	144	add->unmap_ops = kcalloc(count, sizeof(add->unmap_ops[0]), GFP_KERNEL);
142	145	add->kmap_ops = kcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL);
	146	+ add->kunmap_ops = kcalloc(count, sizeof(add->kunmap_ops[0]), GFP_KERNEL);
143	147	add->pages = kcalloc(count, sizeof(add->pages[0]), GFP_KERNEL);
144	148	if (NULL == add->grants \|\|
145	149	NULL == add->map_ops \|\|
146	150	NULL == add->unmap_ops \|\|
147	151	NULL == add->kmap_ops \|\|
	152	+ NULL == add->kunmap_ops \|\|
148	153	NULL == add->pages)
149	154	goto err;
150	155
151		− if (alloc_xenballooned_pages(count, add->pages, false /* lowmem */))
	156	+ if (gnttab_alloc_pages(count, add->pages))
152	157	goto err;
153	158
154	159	for (i = 0; i < count; i++) {
155	160	add->map_ops[i].handle = -1;
156	161	add->unmap_ops[i].handle = -1;
157	162	add->kmap_ops[i].handle = -1;
	163	+ add->kunmap_ops[i].handle = -1;
158	164	}
159	165
160	166	add->index = 0;
@@ -216,9 +222,9 @@ static void gntdev_put_map(struct gntdev_priv priv, struct grant_map map) · 55 unmodified lines
216	222	}
217	223
218	224	if (populate_freeable_maps && priv) {
219		− spin_lock(&priv->lock);
	225	+ mutex_lock(&priv->lock);
220	226	list_del(&map->next);
221		− spin_unlock(&priv->lock);
	227	+ mutex_unlock(&priv->lock);
222	228	}
223	229
224	230	if (map->pages && !use_ptemod)
@@ -239,6 +245,14 @@ static int find_grant_ptes(pte_t *pte, pgtable_t token, · 14 unmodified lines
239	245	BUG_ON(pgnr >= map->count);
240	246	pte_maddr = arbitrary_virt_to_machine(pte).maddr;
241	247
	248	+ /*
	249	+ * Set the PTE as special to force get_user_pages_fast() fall
	250	+ * back to the slow path. If this is not supported as part of
	251	+ * the grant map, it will be done afterwards.
	252	+ */
	253	+ if (xen_feature(XENFEAT_gnttab_map_avail_bits))
	254	+ flags \|= (1 << _GNTMAP_guest_avail0);
	255	+
242	256	gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags,
243	257	map->grants[pgnr].ref,
244	258	map->grants[pgnr].domid);
@@ -247,6 +261,15 @@ static int find_grant_ptes(pte_t *pte, pgtable_t token, · 2 unmodified lines
247	261	return 0;
248	262	}
249	263
	264	+#ifdef CONFIG_X86
	265	+static int set_grant_ptes_as_special(pte_t *pte, pgtable_t token,
	266	+ unsigned long addr, void *data)
	267	+{
	268	+ set_pte_at(current->mm, addr, pte, pte_mkspecial(*pte));
	269	+ return 0;
	270	+}
	271	+#endif
	272	+
250	273	static int map_grant_pages(struct grant_map *map)
251	274	{
252	275	int i, err = 0;
@@ -280,6 +303,8 @@ static int map_grant_pages(struct grant_map *map) · 27 unmodified lines
280	303	map->flags \| GNTMAP_host_map,
281	304	map->grants[i].ref,
282	305	map->grants[i].domid);
	306	+ gnttab_set_unmap_op(&map->kunmap_ops[i], address,
	307	+ map->flags \| GNTMAP_host_map, -1);
283	308	}
284	309	}
285	310
@@ -290,20 +315,42 @@ static int map_grant_pages(struct grant_map *map) · 4 unmodified lines
290	315	return err;
291	316
292	317	for (i = 0; i < map->count; i++) {
293		− if (map->map_ops[i].status)
	318	+ if (map->map_ops[i].status) {
294	319	err = -EINVAL;
295		− else {
296		− BUG_ON(map->map_ops[i].handle == -1);
297		− map->unmap_ops[i].handle = map->map_ops[i].handle;
298		− pr_debug("map handle=%d\n", map->map_ops[i].handle);
	320	+ continue;
299	321	}
	322	+
	323	+ map->unmap_ops[i].handle = map->map_ops[i].handle;
	324	+ if (use_ptemod)
	325	+ map->kunmap_ops[i].handle = map->kmap_ops[i].handle;
300	326	}
301	327	return err;
302	328	}
303	329
	330	+struct unmap_grant_pages_callback_data
	331	+{
	332	+ struct completion completion;
	333	+ int result;
	334	+};
	335	+
	336	+static void unmap_grant_callback(int result,
	337	+ struct gntab_unmap_queue_data *data)
	338	+{
	339	+ struct unmap_grant_pages_callback_data* d = data->data;
	340	+
	341	+ d->result = result;
	342	+ complete(&d->completion);
	343	+}
	344	+
304	345	static int __unmap_grant_pages(struct grant_map *map, int offset, int pages)
305	346	{
306	347	int i, err = 0;
	348	+ struct gntab_unmap_queue_data unmap_data;
	349	+ struct unmap_grant_pages_callback_data data;
	350	+
	351	+ init_completion(&data.completion);
	352	+ unmap_data.data = &data;
	353	+ unmap_data.done= &unmap_grant_callback;
307	354
308	355	if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {
309	356	int pgno = (map->notify.addr >> PAGE_SHIFT);
@@ -315,11 +362,16 @@ static int __unmap_grant_pages(struct grant_map *map, int offset, int pages) · 5 unmodified lines
315	362	}
316	363	}
317	364
318		− err = gnttab_unmap_refs(map->unmap_ops + offset,
319		− use_ptemod ? map->kmap_ops + offset : NULL, map->pages + offset,
320		− pages);
321		− if (err)
322		− return err;
	365	+ unmap_data.unmap_ops = map->unmap_ops + offset;
	366	+ unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;
	367	+ unmap_data.pages = map->pages + offset;
	368	+ unmap_data.count = pages;
	369	+
	370	+ gnttab_unmap_refs_async(&unmap_data);
	371	+
	372	+ wait_for_completion(&data.completion);
	373	+ if (data.result)
	374	+ return data.result;
323	375
324	376	for (i = 0; i < pages; i++) {
325	377	if (map->unmap_ops[offset+i].status)
@@ -387,17 +439,26 @@ static void gntdev_vma_close(struct vm_area_struct *vma) · 61 unmodified lines
387	439	* not do any unmapping, since that has been done prior to
388	440	* closing the vma, but it may still iterate the unmap_ops list.
389	441	*/
390		− spin_lock(&priv->lock);
	442	+ mutex_lock(&priv->lock);
391	443	map->vma = NULL;
392		− spin_unlock(&priv->lock);
	444	+ mutex_unlock(&priv->lock);
393	445	}
394	446	vma->vm_private_data = NULL;
395	447	gntdev_put_map(priv, map);
396	448	}
397	449
	450	+static struct page gntdev_vma_find_special_page(struct vm_area_struct vma,
	451	+ unsigned long addr)
	452	+{
	453	+ struct grant_map *map = vma->vm_private_data;
	454	+
	455	+ return map->pages[(addr - map->pages_vm_start) >> PAGE_SHIFT];
	456	+}
	457	+
398	458	static struct vm_operations_struct gntdev_vmops = {
399	459	.open = gntdev_vma_open,
400	460	.close = gntdev_vma_close,
	461	+ .find_special_page = gntdev_vma_find_special_page,
401	462	};
402	463
403	464	/* ------------------------------------------------------------------ */
@@ -433,14 +494,14 @@ static void mn_invl_range_start(struct mmu_notifier *mn, · 29 unmodified lines
433	494	struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn);
434	495	struct grant_map *map;
435	496
436		− spin_lock(&priv->lock);
	497	+ mutex_lock(&priv->lock);
437	498	list_for_each_entry(map, &priv->maps, next) {
438	499	unmap_if_in_range(map, start, end);
439	500	}
440	501	list_for_each_entry(map, &priv->freeable_maps, next) {
441	502	unmap_if_in_range(map, start, end);
442	503	}
443		− spin_unlock(&priv->lock);
	504	+ mutex_unlock(&priv->lock);
444	505	}
445	506
446	507	static void mn_invl_page(struct mmu_notifier *mn,
@@ -457,7 +518,7 @@ static void mn_release(struct mmu_notifier *mn, · 10 unmodified lines
457	518	struct grant_map *map;
458	519	int err;
459	520
460		− spin_lock(&priv->lock);
	521	+ mutex_lock(&priv->lock);
461	522	list_for_each_entry(map, &priv->maps, next) {
462	523	if (!map->vma)
463	524	continue;
@@ -476,7 +537,7 @@ static void mn_release(struct mmu_notifier *mn, · 12 unmodified lines
476	537	err = unmap_grant_pages(map, /* offset */ 0, map->count);
477	538	WARN_ON(err);
478	539	}
479		− spin_unlock(&priv->lock);
	540	+ mutex_unlock(&priv->lock);
480	541	}
481	542
482	543	static struct mmu_notifier_ops gntdev_mmu_ops = {
@@ -498,7 +559,7 @@ static int gntdev_open(struct inode inode, struct file flip) · 15 unmodified lines
498	559
499	560	INIT_LIST_HEAD(&priv->maps);
500	561	INIT_LIST_HEAD(&priv->freeable_maps);
501		− spin_lock_init(&priv->lock);
	562	+ mutex_init(&priv->lock);
502	563
503	564	if (use_ptemod) {
504	565	priv->mm = get_task_mm(current);
@@ -572,10 +633,10 @@ static long gntdev_ioctl_map_grant_ref(struct gntdev_priv *priv, · 67 unmodified lines
572	633	return -EFAULT;
573	634	}
574	635
575		− spin_lock(&priv->lock);
	636	+ mutex_lock(&priv->lock);
576	637	gntdev_add_map(priv, map);
577	638	op.index = map->index << PAGE_SHIFT;
578		− spin_unlock(&priv->lock);
	639	+ mutex_unlock(&priv->lock);
579	640
580	641	if (copy_to_user(u, &op, sizeof(op)) != 0)
581	642	return -EFAULT;
@@ -594,7 +655,7 @@ static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv *priv, · 12 unmodified lines
594	655	return -EFAULT;
595	656	pr_debug("priv %p, del %d+%d\n", priv, (int)op.index, (int)op.count);
596	657
597		− spin_lock(&priv->lock);
	658	+ mutex_lock(&priv->lock);
598	659	map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count);
599	660	if (map) {
600	661	list_del(&map->next);
@@ -602,7 +663,7 @@ static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv *priv, · 1 unmodified line
602	663	list_add_tail(&map->next, &priv->freeable_maps);
603	664	err = 0;
604	665	}
605		− spin_unlock(&priv->lock);
	666	+ mutex_unlock(&priv->lock);
606	667	if (map)
607	668	gntdev_put_map(priv, map);
608	669	return err;
@@ -670,7 +731,7 @@ static long gntdev_ioctl_notify(struct gntdev_priv priv, void __user u) · 61 unmodified lines
670	731	out_flags = op.action;
671	732	out_event = op.event_channel_port;
672	733
673		− spin_lock(&priv->lock);
	734	+ mutex_lock(&priv->lock);
674	735
675	736	list_for_each_entry(map, &priv->maps, next) {
676	737	uint64_t begin = map->index << PAGE_SHIFT;
@@ -698,7 +759,7 @@ static long gntdev_ioctl_notify(struct gntdev_priv priv, void __user u) · 21 unmodified lines
698	759	rc = 0;
699	760
700	761	unlock_out:
701		− spin_unlock(&priv->lock);
	762	+ mutex_unlock(&priv->lock);
702	763
703	764	/* Drop the reference to the event channel we did not save in the map */
704	765	if (out_flags & UNMAP_NOTIFY_SEND_EVENT)
@@ -748,7 +809,7 @@ static int gntdev_mmap(struct file flip, struct vm_area_struct vma) · 43 unmodified lines
748	809	pr_debug("map %d+%d at %lx (pgoff %lx)\n",
749	810	index, count, vma->vm_start, vma->vm_pgoff);
750	811
751		− spin_lock(&priv->lock);
	812	+ mutex_lock(&priv->lock);
752	813	map = gntdev_find_map_index(priv, index, count);
753	814	if (!map)
754	815	goto unlock_out;
@@ -783,7 +844,7 @@ static int gntdev_mmap(struct file flip, struct vm_area_struct vma) · 28 unmodified lines
783	844	map->flags \|= GNTMAP_readonly;
784	845	}
785	846
786		− spin_unlock(&priv->lock);
	847	+ mutex_unlock(&priv->lock);
787	848
788	849	if (use_ptemod) {
789	850	err = apply_to_page_range(vma->vm_mm, vma->vm_start,
@@ -806,16 +867,34 @@ static int gntdev_mmap(struct file flip, struct vm_area_struct vma) · 16 unmodified lines
806	867	if (err)
807	868	goto out_put_map;
808	869	}
	870	+ } else {
	871	+#ifdef CONFIG_X86
	872	+ /*
	873	+ * If the PTEs were not made special by the grant map
	874	+ * hypercall, do so here.
	875	+ *
	876	+ * This is racy since the mapping is already visible
	877	+ * to userspace but userspace should be well-behaved
	878	+ * enough to not touch it until the mmap() call
	879	+ * returns.
	880	+ */
	881	+ if (!xen_feature(XENFEAT_gnttab_map_avail_bits)) {
	882	+ apply_to_page_range(vma->vm_mm, vma->vm_start,
	883	+ vma->vm_end - vma->vm_start,
	884	+ set_grant_ptes_as_special, NULL);
	885	+ }
	886	+#endif
	887	+ map->pages_vm_start = vma->vm_start;
809	888	}
810	889
811	890	return 0;
812	891
813	892	unlock_out:
814		− spin_unlock(&priv->lock);
	893	+ mutex_unlock(&priv->lock);
815	894	return err;
816	895
817	896	out_unlock_put:
818		− spin_unlock(&priv->lock);
	897	+ mutex_unlock(&priv->lock);
819	898	out_put_map:
820	899	if (use_ptemod)
821	900	map->vma = NULL;

drivers/xen/grant-table.c+116−4

@@ -42,6 +42,7 @@ · 41 unmodified lines
42	42	#include <linux/io.h>
43	43	#include <linux/delay.h>
44	44	#include <linux/hardirq.h>
	45	+#include <linux/workqueue.h>
45	46
46	47	#include <xen/xen.h>
47	48	#include <xen/interface/xen.h>
@@ -50,6 +51,7 @@ · 2 unmodified lines
50	51	#include <xen/interface/memory.h>
51	52	#include <xen/hvc-console.h>
52	53	#include <xen/swiotlb-xen.h>
	54	+#include <xen/balloon.h>
53	55	#include <asm/xen/hypercall.h>
54	56	#include <asm/xen/interface.h>
55	57
@@ -671,6 +673,59 @@ void gnttab_free_auto_xlat_frames(void) · 615 unmodified lines
671	673	}
672	674	EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);
673	675
	676	+/**
	677	+ * gnttab_alloc_pages - alloc pages suitable for grant mapping into
	678	+ * @nr_pages: number of pages to alloc
	679	+ * @pages: returns the pages
	680	+ */
	681	+int gnttab_alloc_pages(int nr_pages, struct page **pages)
	682	+{
	683	+ int i;
	684	+ int ret;
	685	+
	686	+ ret = alloc_xenballooned_pages(nr_pages, pages, false);
	687	+ if (ret < 0)
	688	+ return ret;
	689	+
	690	+ for (i = 0; i < nr_pages; i++) {
	691	+#if BITS_PER_LONG < 64
	692	+ struct xen_page_foreign *foreign;
	693	+
	694	+ foreign = kzalloc(sizeof(*foreign), GFP_KERNEL);
	695	+ if (!foreign) {
	696	+ gnttab_free_pages(nr_pages, pages);
	697	+ return -ENOMEM;
	698	+ }
	699	+ set_page_private(pages[i], (unsigned long)foreign);
	700	+#endif
	701	+ SetPagePrivate(pages[i]);
	702	+ }
	703	+
	704	+ return 0;
	705	+}
	706	+EXPORT_SYMBOL(gnttab_alloc_pages);
	707	+
	708	+/**
	709	+ * gnttab_free_pages - free pages allocated by gnttab_alloc_pages()
	710	+ * @nr_pages; number of pages to free
	711	+ * @pages: the pages
	712	+ */
	713	+void gnttab_free_pages(int nr_pages, struct page **pages)
	714	+{
	715	+ int i;
	716	+
	717	+ for (i = 0; i < nr_pages; i++) {
	718	+ if (PagePrivate(pages[i])) {
	719	+#if BITS_PER_LONG < 64
	720	+ kfree((void *)page_private(pages[i]));
	721	+#endif
	722	+ ClearPagePrivate(pages[i]);
	723	+ }
	724	+ }
	725	+ free_xenballooned_pages(nr_pages, pages);
	726	+}
	727	+EXPORT_SYMBOL(gnttab_free_pages);
	728	+
674	729	/* Handling of paged out grant targets (GNTST_eagain) */
675	730	#define MAX_DELAY 256
676	731	static inline void
@@ -727,30 +782,87 @@ int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, · 50 unmodified lines
727	782	if (ret)
728	783	return ret;
729	784
730		− /* Retry eagain maps */
731		− for (i = 0; i < count; i++)
	785	+ for (i = 0; i < count; i++) {
	786	+ /* Retry eagain maps */
732	787	if (map_ops[i].status == GNTST_eagain)
733	788	gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
734	789	&map_ops[i].status, __func__);
735	790
	791	+ if (map_ops[i].status == GNTST_okay) {
	792	+ struct xen_page_foreign *foreign;
	793	+
	794	+ SetPageForeign(pages[i]);
	795	+ foreign = xen_page_foreign(pages[i]);
	796	+ foreign->domid = map_ops[i].dom;
	797	+ foreign->gref = map_ops[i].ref;
	798	+ }
	799	+ }
	800	+
736	801	return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count);
737	802	}
738	803	EXPORT_SYMBOL_GPL(gnttab_map_refs);
739	804
740	805	int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
741		− struct gnttab_map_grant_ref *kmap_ops,
	806	+ struct gnttab_unmap_grant_ref *kunmap_ops,
742	807	struct page **pages, unsigned int count)
743	808	{
	809	+ unsigned int i;
744	810	int ret;
745	811
746	812	ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
747	813	if (ret)
748	814	return ret;
749	815
750		− return clear_foreign_p2m_mapping(unmap_ops, kmap_ops, pages, count);
	816	+ for (i = 0; i < count; i++)
	817	+ ClearPageForeign(pages[i]);
	818	+
	819	+ return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count);
751	820	}
752	821	EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
753	822
	823	+#define GNTTAB_UNMAP_REFS_DELAY 5
	824	+
	825	+static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item);
	826	+
	827	+static void gnttab_unmap_work(struct work_struct *work)
	828	+{
	829	+ struct gntab_unmap_queue_data
	830	+ *unmap_data = container_of(work,
	831	+ struct gntab_unmap_queue_data,
	832	+ gnttab_work.work);
	833	+ if (unmap_data->age != UINT_MAX)
	834	+ unmap_data->age++;
	835	+ __gnttab_unmap_refs_async(unmap_data);
	836	+}
	837	+
	838	+static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
	839	+{
	840	+ int ret;
	841	+ int pc;
	842	+
	843	+ for (pc = 0; pc < item->count; pc++) {
	844	+ if (page_count(item->pages[pc]) > 1) {
	845	+ unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1);
	846	+ schedule_delayed_work(&item->gnttab_work,
	847	+ msecs_to_jiffies(delay));
	848	+ return;
	849	+ }
	850	+ }
	851	+
	852	+ ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops,
	853	+ item->pages, item->count);
	854	+ item->done(ret, item);
	855	+}
	856	+
	857	+void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
	858	+{
	859	+ INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work);
	860	+ item->age = 0;
	861	+
	862	+ __gnttab_unmap_refs_async(item);
	863	+}
	864	+EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async);
	865	+
754	866	static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
755	867	{
756	868	int rc;

drivers/xen/manage.c+7−1

@@ -105,10 +105,16 @@ static void do_suspend(void) · 104 unmodified lines
105	105
106	106	err = freeze_processes();
107	107	if (err) {
108		− pr_err("%s: freeze failed %d\n", __func__, err);
	108	+ pr_err("%s: freeze processes failed %d\n", __func__, err);
109	109	goto out;
110	110	}
111	111
	112	+ err = freeze_kernel_threads();
	113	+ if (err) {
	114	+ pr_err("%s: freeze kernel threads failed %d\n", __func__, err);
	115	+ goto out_thaw;
	116	+ }
	117	+
112	118	err = dpm_suspend_start(PMSG_FREEZE);
113	119	if (err) {
114	120	pr_err("%s: dpm_suspend_start %d\n", __func__, err);

drivers/xen/tmem.c+1−1

@@ -374,7 +374,7 @@ static struct frontswap_ops tmem_frontswap_ops = { · 373 unmodified lines
374	374	};
375	375	#endif
376	376
377		−static int xen_tmem_init(void)
	377	+static int __init xen_tmem_init(void)
378	378	{
379	379	if (!xen_domain())
380	380	return 0;

drivers/xen/xen-scsiback.c+3−3

@@ -227,7 +227,7 @@ static void put_free_pages(struct page **page, int num) · 226 unmodified lines
227	227	return;
228	228	if (i > scsiback_max_buffer_pages) {
229	229	n = min(num, i - scsiback_max_buffer_pages);
230		− free_xenballooned_pages(n, page + num - n);
	230	+ gnttab_free_pages(n, page + num - n);
231	231	n = num - n;
232	232	}
233	233	spin_lock_irqsave(&free_pages_lock, flags);
@@ -244,7 +244,7 @@ static int get_free_page(struct page **page) · 10 unmodified lines
244	244	spin_lock_irqsave(&free_pages_lock, flags);
245	245	if (list_empty(&scsiback_free_pages)) {
246	246	spin_unlock_irqrestore(&free_pages_lock, flags);
247		− return alloc_xenballooned_pages(1, page, false);
	247	+ return gnttab_alloc_pages(1, page);
248	248	}
249	249	page[0] = list_first_entry(&scsiback_free_pages, struct page, lru);
250	250	list_del(&page[0]->lru);
@@ -2106,7 +2106,7 @@ static void __exit scsiback_exit(void) · 1855 unmodified lines
2106	2106	while (free_pages_num) {
2107	2107	if (get_free_page(&page))
2108	2108	BUG();
2109		− free_xenballooned_pages(1, &page);
	2109	+ gnttab_free_pages(1, &page);
2110	2110	}
2111	2111	scsiback_deregister_configfs();
2112	2112	xenbus_unregister_driver(&scsiback_driver);

drivers/xen/xenbus/xenbus_dev_frontend.c+7−4

@@ -326,10 +326,13 @@ static int xenbus_write_transaction(unsigned msg_type, · 325 unmodified lines
326	326	}
327	327
328	328	if (msg_type == XS_TRANSACTION_START) {
329		− trans->handle.id = simple_strtoul(reply, NULL, 0);
330		−
331		− list_add(&trans->list, &u->transactions);
332		− } else if (msg_type == XS_TRANSACTION_END) {
	329	+ if (u->u.msg.type == XS_ERROR)
	330	+ kfree(trans);
	331	+ else {
	332	+ trans->handle.id = simple_strtoul(reply, NULL, 0);
	333	+ list_add(&trans->list, &u->transactions);
	334	+ }
	335	+ } else if (u->u.msg.type == XS_TRANSACTION_END) {
333	336	list_for_each_entry(trans, &u->transactions, list)
334	337	if (trans->handle.id == u->u.msg.tx_id)
335	338	break;

include/linux/mm.h+8−0

@@ -290,6 +290,14 @@ struct vm_operations_struct { · 289 unmodified lines
290	290	/* called by sys_remap_file_pages() to populate non-linear mapping */
291	291	int (remap_pages)(struct vm_area_struct vma, unsigned long addr,
292	292	unsigned long size, pgoff_t pgoff);
	293	+
	294	+ /*
	295	+ * Called by vm_normal_page() for special PTEs to find the
	296	+ * page for @addr. This is useful if the default behavior
	297	+ * (using pte_page()) would not find the correct page.
	298	+ */
	299	+ struct page (find_special_page)(struct vm_area_struct *vma,
	300	+ unsigned long addr);
293	301	};
294	302
295	303	struct mmu_gather;

include/linux/page-flags.h+5−0

@@ -121,8 +121,12 @@ enum pageflags { · 120 unmodified lines
121	121	PG_fscache = PG_private_2, /* page backed by cache */
122	122
123	123	/* XEN */
	124	+ /* Pinned in Xen as a read-only pagetable page. */
124	125	PG_pinned = PG_owner_priv_1,
	126	+ /* Pinned as part of domain save (see xen_mm_pin_all()). */
125	127	PG_savepinned = PG_dirty,
	128	+ /* Has a grant mapping of another (foreign) domain's page. */
	129	+ PG_foreign = PG_owner_priv_1,
126	130
127	131	/* SLOB */
128	132	PG_slob_free = PG_private,
@@ -215,6 +219,7 @@ __PAGEFLAG(Slab, slab) · 86 unmodified lines
215	219	PAGEFLAG(Checked, checked) /* Used by some filesystems */
216	220	PAGEFLAG(Pinned, pinned) TESTSCFLAG(Pinned, pinned) /* Xen */
217	221	PAGEFLAG(SavePinned, savepinned); /* Xen */
	222	+PAGEFLAG(Foreign, foreign); /* Xen */
218	223	PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved)
219	224	PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked)
220	225	__SETPAGEFLAG(SwapBacked, swapbacked)

include/xen/grant_table.h+42−1

@@ -45,6 +45,8 @@ · 44 unmodified lines
45	45	#include <asm/xen/hypervisor.h>
46	46
47	47	#include <xen/features.h>
	48	+#include <linux/mm_types.h>
	49	+#include <linux/page-flags.h>
48	50
49	51	#define GNTTAB_RESERVED_XENSTORE 1
50	52
@@ -58,6 +60,22 @@ struct gnttab_free_callback { · 7 unmodified lines
58	60	u16 count;
59	61	};
60	62
	63	+struct gntab_unmap_queue_data;
	64	+
	65	+typedef void (gnttab_unmap_refs_done)(int result, struct gntab_unmap_queue_data data);
	66	+
	67	+struct gntab_unmap_queue_data
	68	+{
	69	+ struct delayed_work gnttab_work;
	70	+ void *data;
	71	+ gnttab_unmap_refs_done done;
	72	+ struct gnttab_unmap_grant_ref *unmap_ops;
	73	+ struct gnttab_unmap_grant_ref *kunmap_ops;
	74	+ struct page **pages;
	75	+ unsigned int count;
	76	+ unsigned int age;
	77	+};
	78	+
61	79	int gnttab_init(void);
62	80	int gnttab_suspend(void);
63	81	int gnttab_resume(void);
@@ -163,12 +181,17 @@ void gnttab_free_auto_xlat_frames(void); · 99 unmodified lines
163	181
164	182	#define gnttab_map_vaddr(map) ((void *)(map.host_virt_addr))
165	183
	184	+int gnttab_alloc_pages(int nr_pages, struct page **pages);
	185	+void gnttab_free_pages(int nr_pages, struct page **pages);
	186	+
166	187	int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
167	188	struct gnttab_map_grant_ref *kmap_ops,
168	189	struct page **pages, unsigned int count);
169	190	int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
170		− struct gnttab_map_grant_ref *kunmap_ops,
	191	+ struct gnttab_unmap_grant_ref *kunmap_ops,
171	192	struct page **pages, unsigned int count);
	193	+void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item);
	194	+
172	195
173	196	/* Perform a batch of grant map/copy operations. Retry every batch slot
174	197	* for which the hypervisor returns GNTST_eagain. This is typically due
@@ -182,4 +205,22 @@ int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, · 7 unmodified lines
182	205	void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count);
183	206	void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count);
184	207
	208	+
	209	+struct xen_page_foreign {
	210	+ domid_t domid;
	211	+ grant_ref_t gref;
	212	+};
	213	+
	214	+static inline struct xen_page_foreign xen_page_foreign(struct page page)
	215	+{
	216	+ if (!PageForeign(page))
	217	+ return NULL;
	218	+#if BITS_PER_LONG < 64
	219	+ return (struct xen_page_foreign *)page->private;
	220	+#else
	221	+ BUILD_BUG_ON(sizeof(struct xen_page_foreign) > BITS_PER_LONG);
	222	+ return (struct xen_page_foreign *)&page->private;
	223	+#endif
	224	+}
	225	+
185	226	#endif /* __ASM_GNTTAB_H__ */

include/xen/interface/features.h+6−0

@@ -41,6 +41,12 @@ · 40 unmodified lines
41	41	/* x86: Does this Xen host support the MMU_PT_UPDATE_PRESERVE_AD hypercall? */
42	42	#define XENFEAT_mmu_pt_update_preserve_ad 5
43	43
	44	+/*
	45	+ * If set, GNTTABOP_map_grant_ref honors flags to be placed into guest kernel
	46	+ * available pte bits.
	47	+ */
	48	+#define XENFEAT_gnttab_map_avail_bits 7
	49	+
44	50	/* x86: Does this Xen host support the HVM callback vector type? */
45	51	#define XENFEAT_hvm_callback_vector 8
46	52

include/xen/interface/grant_table.h+7−0

@@ -525,6 +525,13 @@ DEFINE_GUEST_HANDLE_STRUCT(gnttab_cache_flush); · 524 unmodified lines
525	525	#define _GNTMAP_contains_pte (4)
526	526	#define GNTMAP_contains_pte (1<<_GNTMAP_contains_pte)
527	527
	528	+/*
	529	+ * Bits to be placed in guest kernel available PTE bits (architecture
	530	+ * dependent; only supported when XENFEAT_gnttab_map_avail_bits is set).
	531	+ */
	532	+#define _GNTMAP_guest_avail0 (16)
	533	+#define GNTMAP_guest_avail_mask ((uint32_t)~0 << _GNTMAP_guest_avail0)
	534	+
528	535	/*
529	536	* Values for error status returns. All errors are -ve.
530	537	*/

mm/memory.c+2−0

@@ -754,6 +754,8 @@ struct page vm_normal_page(struct vm_area_struct vma, unsigned long addr, · 753 unmodified lines
754	754	if (HAVE_PTE_SPECIAL) {
755	755	if (likely(!pte_special(pte)))
756	756	goto check_pfn;
	757	+ if (vma->vm_ops && vma->vm_ops->find_special_page)
	758	+ return vma->vm_ops->find_special_page(vma, addr);
757	759	if (vma->vm_flags & (VM_PFNMAP \| VM_MIXEDMAP))
758	760	return NULL;
759	761	if (!is_zero_pfn(pfn))

Merge tag 'stable/for-linus-3.20-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code