/* * Copyright (C) 2001-2006 Silicon Graphics, Inc.  All rights * reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of version 2 of the GNU General Public License * as published by the Free Software Foundation. */ /* * SN Platform Special Memory (mspec) Support * * This driver exports the SN special memory (mspec) facility to user * processes. * There are three types of memory made available thru this driver: * fetchops, uncached and cached. * * Fetchops are atomic memory operations that are implemented in the * memory controller on SGI SN hardware. * * Uncached are used for memory write combining feature of the ia64 * cpu. * * Cached are used for areas of memory that are used as cached addresses * on our partition and used as uncached addresses from other partitions. * Due to a design constraint of the SN2 Shub, you can not have processors * on the same FSB perform both a cached and uncached reference to the * same cache line.  These special memory cached regions prevent the * kernel from ever dropping in a TLB entry and therefore prevent the * processor from ever speculating a cache line from this page. */ #include <linux/types.h>#include <linux/kernel.h>#include <linux/module.h>#include <linux/init.h>#include <linux/errno.h>#include <linux/miscdevice.h>#include <linux/spinlock.h>#include <linux/mm.h>#include <linux/vmalloc.h>#include <linux/string.h>#include <linux/slab.h>#include <linux/numa.h>#include <asm/page.h>#include <asm/system.h>#include <asm/pgtable.h>#include <asm/atomic.h>#include <asm/tlbflush.h>#include <asm/uncached.h>#include <asm/sn/addrs.h>#include <asm/sn/arch.h>#include <asm/sn/mspec.h>#include <asm/sn/sn_cpuid.h>#include <asm/sn/io.h>#include <asm/sn/bte.h>#include <asm/sn/shubio.h>  #define FETCHOP_ID	"SGI Fetchop,"#define CACHED_ID	"Cached,"#define UNCACHED_ID	"Uncached"#define REVISION	"4.0"#define MSPEC_BASENAME	"mspec" /* * Page types allocated by the device. */enum {	MSPEC_FETCHOP = 1,	MSPEC_CACHED,	MSPEC_UNCACHED}; #ifdef CONFIG_SGI_SNstatic int is_sn2;#else#define is_sn2		0#endif /* * One of these structures is allocated when an mspec region is mmaped. The * structure is pointed to by the vma->vm_private_data field in the vma struct. * This structure is used to record the addresses of the mspec pages. */struct vma_data {	atomic_t refcnt;	/* Number of vmas sharing the data. */	spinlock_t lock;	/* Serialize access to the vma. */	int count;		/* Number of pages allocated. */	int type;		/* Type of pages allocated. */	unsigned long maddr[0];	/* Array of MSPEC addresses. */}; /* used on shub2 to clear FOP cache in the HUB */static unsigned long scratch_page[MAX_NUMNODES];#define SH2_AMO_CACHE_ENTRIES	4 static inline intmspec_zero_block(unsigned long addr, int len){	int status; 	if (is_sn2) {		if (is_shub2()) {			int nid;			void *p;			int i; 			nid = nasid_to_cnodeid(get_node_number(__pa(addr)));			p = (void *)TO_AMO(scratch_page[nid]); 			for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) {				FETCHOP_LOAD_OP(p, FETCHOP_LOAD);				p += FETCHOP_VAR_SIZE;			}		} 		status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len,				  BTE_WACQUIRE | BTE_ZERO_FILL, NULL);	} else {		memset((char *) addr, 0, len);		status = 0;	}	return status;} /* * mspec_open * * Called when a device mapping is created by a means other than mmap * (via fork, etc.).  Increments the reference count on the underlying * mspec data so it is not freed prematurely. */static voidmspec_open(struct vm_area_struct *vma){	struct vma_data *vdata; 	vdata = vma->vm_private_data;	atomic_inc(&vdata->refcnt);} /* * mspec_close * * Called when unmapping a device mapping. Frees all mspec pages * belonging to the vma. */static voidmspec_close(struct vm_area_struct *vma){	struct vma_data *vdata;	int i, pages, result, vdata_size; 	vdata = vma->vm_private_data;	if (!atomic_dec_and_test(&vdata->refcnt))		return; 	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);	for (i = 0; i < pages; i++) {		if (vdata->maddr[i] == 0)			continue;		/*		 * Clear the page before sticking it back		 * into the pool.		 */		result = mspec_zero_block(vdata->maddr[i], PAGE_SIZE);		if (!result)			uncached_free_page(vdata->maddr[i]);		else			printk(KERN_WARNING "mspec_close(): "			       "failed to zero page %i\n",			       result);	} 	if (vdata_size <= PAGE_SIZE)		kfree(vdata);	else		vfree(vdata);}  /* * mspec_nopfn * * Creates a mspec page and maps it to user space. */static unsigned longmspec_nopfn(struct vm_area_struct *vma, unsigned long address){	unsigned long paddr, maddr;	unsigned long pfn;	int index;	struct vma_data *vdata = vma->vm_private_data; 	index = (address - vma->vm_start) >> PAGE_SHIFT;	maddr = (volatile unsigned long) vdata->maddr[index];	if (maddr == 0) {		maddr = uncached_alloc_page(numa_node_id());		if (maddr == 0)			return NOPFN_OOM; 		spin_lock(&vdata->lock);		if (vdata->maddr[index] == 0) {			vdata->count++;			vdata->maddr[index] = maddr;		} else {			uncached_free_page(maddr);			maddr = vdata->maddr[index];		}		spin_unlock(&vdata->lock);	} 	if (vdata->type == MSPEC_FETCHOP)		paddr = TO_AMO(maddr);	else		paddr = maddr & ~__IA64_UNCACHED_OFFSET; 	pfn = paddr >> PAGE_SHIFT; 	return pfn;} static struct vm_operations_struct mspec_vm_ops = {	.open = mspec_open,	.close = mspec_close,	.nopfn = mspec_nopfn}; /* * mspec_mmap * * Called when mmaping the device.  Initializes the vma with a fault handler * and private data structure necessary to allocate, track, and free the * underlying pages. */static intmspec_mmap(struct file *file, struct vm_area_struct *vma, int type){	struct vma_data *vdata;	int pages, vdata_size; 	if (vma->vm_pgoff != 0)		return -EINVAL; 	if ((vma->vm_flags & VM_SHARED) == 0)		return -EINVAL; 	if ((vma->vm_flags & VM_WRITE) == 0)		return -EPERM; 	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);	if (vdata_size <= PAGE_SIZE)		vdata = kmalloc(vdata_size, GFP_KERNEL);	else		vdata = vmalloc(vdata_size);	if (!vdata)		return -ENOMEM;	memset(vdata, 0, vdata_size); 	vdata->type = type;	spin_lock_init(&vdata->lock);	vdata->refcnt = ATOMIC_INIT(1);	vma->vm_private_data = vdata; 	vma->vm_flags |= (VM_IO | VM_LOCKED | VM_RESERVED | VM_PFNMAP);	if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED)		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);	vma->vm_ops = &mspec_vm_ops; 	return 0;} static intfetchop_mmap(struct file *file, struct vm_area_struct *vma){	return mspec_mmap(file, vma, MSPEC_FETCHOP);} static intcached_mmap(struct file *file, struct vm_area_struct *vma){	return mspec_mmap(file, vma, MSPEC_CACHED);} static intuncached_mmap(struct file *file, struct vm_area_struct *vma){	return mspec_mmap(file, vma, MSPEC_UNCACHED);} static const struct file_operations fetchop_fops = {	.owner = THIS_MODULE,	.mmap = fetchop_mmap}; static struct miscdevice fetchop_miscdev = {	.minor = MISC_DYNAMIC_MINOR,	.name = "sgi_fetchop",	.fops = &fetchop_fops}; static const struct file_operations cached_fops = {	.owner = THIS_MODULE,	.mmap = cached_mmap}; static struct miscdevice cached_miscdev = {	.minor = MISC_DYNAMIC_MINOR,	.name = "mspec_cached",	.fops = &cached_fops}; static const struct file_operations uncached_fops = {	.owner = THIS_MODULE,	.mmap = uncached_mmap}; static struct miscdevice uncached_miscdev = {	.minor = MISC_DYNAMIC_MINOR,	.name = "mspec_uncached",	.fops = &uncached_fops}; /* * mspec_init * * Called at boot time to initialize the mspec facility. */static int __initmspec_init(void){	int ret;	int nid; 	/*	 * The fetchop device only works on SN2 hardware, uncached and cached	 * memory drivers should both be valid on all ia64 hardware	 */#ifdef CONFIG_SGI_SN	if (ia64_platform_is("sn2")) {		is_sn2 = 1;		if (is_shub2()) {			ret = -ENOMEM;			for_each_online_node(nid) {				int actual_nid;				int nasid;				unsigned long phys; 				scratch_page[nid] = uncached_alloc_page(nid);				if (scratch_page[nid] == 0)					goto free_scratch_pages;				phys = __pa(scratch_page[nid]);				nasid = get_node_number(phys);				actual_nid = nasid_to_cnodeid(nasid);				if (actual_nid != nid)					goto free_scratch_pages;			}		} 		ret = misc_register(&fetchop_miscdev);		if (ret) {			printk(KERN_ERR			       "%s: failed to register device %i\n",			       FETCHOP_ID, ret);			goto free_scratch_pages;		}	}#endif	ret = misc_register(&cached_miscdev);	if (ret) {		printk(KERN_ERR "%s: failed to register device %i\n",		       CACHED_ID, ret);		if (is_sn2)			misc_deregister(&fetchop_miscdev);		goto free_scratch_pages;	}	ret = misc_register(&uncached_miscdev);	if (ret) {		printk(KERN_ERR "%s: failed to register device %i\n",		       UNCACHED_ID, ret);		misc_deregister(&cached_miscdev);		if (is_sn2)			misc_deregister(&fetchop_miscdev);		goto free_scratch_pages;	} 	printk(KERN_INFO "%s %s initialized devices: %s %s %s\n",	       MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "",	       CACHED_ID, UNCACHED_ID); 	return 0;  free_scratch_pages:	for_each_node(nid) {		if (scratch_page[nid] != 0)			uncached_free_page(scratch_page[nid]);	}	return ret;} static void __exitmspec_exit(void){	int nid; 	misc_deregister(&uncached_miscdev);	misc_deregister(&cached_miscdev);	if (is_sn2) {		misc_deregister(&fetchop_miscdev); 		for_each_node(nid) {			if (scratch_page[nid] != 0)				uncached_free_page(scratch_page[nid]);		}	}} module_init(mspec_init);module_exit(mspec_exit); MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");MODULE_DESCRIPTION("Driver for SGI SN special memory operations");MODULE_LICENSE("GPL");