linux/drivers/misc/sgi-xp/xpc_sn2.c

2463 lines
68 KiB
C
Raw Normal View History

/*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
* Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved.
*/
/*
* Cross Partition Communication (XPC) sn2-based functions.
*
* Architecture specific implementation of common functions.
*
*/
#include <linux/delay.h>
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h percpu.h is included by sched.h and module.h and thus ends up being included when building most .c files. percpu.h includes slab.h which in turn includes gfp.h making everything defined by the two files universally available and complicating inclusion dependencies. percpu.h -> slab.h dependency is about to be removed. Prepare for this change by updating users of gfp and slab facilities include those headers directly instead of assuming availability. As this conversion needs to touch large number of source files, the following script is used as the basis of conversion. http://userweb.kernel.org/~tj/misc/slabh-sweep.py The script does the followings. * Scan files for gfp and slab usages and update includes such that only the necessary includes are there. ie. if only gfp is used, gfp.h, if slab is used, slab.h. * When the script inserts a new include, it looks at the include blocks and try to put the new include such that its order conforms to its surrounding. It's put in the include block which contains core kernel includes, in the same order that the rest are ordered - alphabetical, Christmas tree, rev-Xmas-tree or at the end if there doesn't seem to be any matching order. * If the script can't find a place to put a new include (mostly because the file doesn't have fitting include block), it prints out an error message indicating which .h file needs to be added to the file. The conversion was done in the following steps. 1. The initial automatic conversion of all .c files updated slightly over 4000 files, deleting around 700 includes and adding ~480 gfp.h and ~3000 slab.h inclusions. The script emitted errors for ~400 files. 2. Each error was manually checked. Some didn't need the inclusion, some needed manual addition while adding it to implementation .h or embedding .c file was more appropriate for others. This step added inclusions to around 150 files. 3. The script was run again and the output was compared to the edits from #2 to make sure no file was left behind. 4. Several build tests were done and a couple of problems were fixed. e.g. lib/decompress_*.c used malloc/free() wrappers around slab APIs requiring slab.h to be added manually. 5. The script was run on all .h files but without automatically editing them as sprinkling gfp.h and slab.h inclusions around .h files could easily lead to inclusion dependency hell. Most gfp.h inclusion directives were ignored as stuff from gfp.h was usually wildly available and often used in preprocessor macros. Each slab.h inclusion directive was examined and added manually as necessary. 6. percpu.h was updated not to include slab.h. 7. Build test were done on the following configurations and failures were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my distributed build env didn't work with gcov compiles) and a few more options had to be turned off depending on archs to make things build (like ipr on powerpc/64 which failed due to missing writeq). * x86 and x86_64 UP and SMP allmodconfig and a custom test config. * powerpc and powerpc64 SMP allmodconfig * sparc and sparc64 SMP allmodconfig * ia64 SMP allmodconfig * s390 SMP allmodconfig * alpha SMP allmodconfig * um on x86_64 SMP allmodconfig 8. percpu.h modifications were reverted so that it could be applied as a separate patch and serve as bisection point. Given the fact that I had only a couple of failures from tests on step 6, I'm fairly confident about the coverage of this conversion patch. If there is a breakage, it's likely to be something in one of the arch headers which should be easily discoverable easily on most builds of the specific arch. Signed-off-by: Tejun Heo <tj@kernel.org> Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org> Cc: Ingo Molnar <mingo@redhat.com> Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 08:04:11 +00:00
#include <linux/slab.h>
#include <asm/uncached.h>
#include <asm/sn/mspec.h>
#include <asm/sn/sn_sal.h>
#include "xpc.h"
/*
* Define the number of u64s required to represent all the C-brick nasids
* as a bitmap. The cross-partition kernel modules deal only with
* C-brick nasids, thus the need for bitmaps which don't account for
* odd-numbered (non C-brick) nasids.
*/
#define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2)
#define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8)
#define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64)
/*
* Memory for XPC's amo variables is allocated by the MSPEC driver. These
* pages are located in the lowest granule. The lowest granule uses 4k pages
* for cached references and an alternate TLB handler to never provide a
* cacheable mapping for the entire region. This will prevent speculative
* reading of cached copies of our lines from being issued which will cause
* a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64
* amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of
* NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify
* the senders of ACTIVATE IRQs, 1 amo variable to identify which remote
* partitions (i.e., XPCs) consider themselves currently engaged with the
* local XPC and 1 amo variable to request partition deactivation.
*/
#define XPC_NOTIFY_IRQ_AMOS_SN2 0
#define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \
XP_MAX_NPARTITIONS_SN2)
#define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \
XP_NASID_MASK_WORDS_SN2)
#define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1)
/*
* Buffer used to store a local copy of portions of a remote partition's
* reserved page (either its header and part_nasids mask, or its vars).
*/
static void *xpc_remote_copy_buffer_base_sn2;
static char *xpc_remote_copy_buffer_sn2;
static struct xpc_vars_sn2 *xpc_vars_sn2;
static struct xpc_vars_part_sn2 *xpc_vars_part_sn2;
static int
xpc_setup_partitions_sn2(void)
{
/* nothing needs to be done */
return 0;
}
static void
xpc_teardown_partitions_sn2(void)
{
/* nothing needs to be done */
}
/* SH_IPI_ACCESS shub register value on startup */
static u64 xpc_sh1_IPI_access_sn2;
static u64 xpc_sh2_IPI_access0_sn2;
static u64 xpc_sh2_IPI_access1_sn2;
static u64 xpc_sh2_IPI_access2_sn2;
static u64 xpc_sh2_IPI_access3_sn2;
/*
* Change protections to allow IPI operations.
*/
static void
xpc_allow_IPI_ops_sn2(void)
{
int node;
int nasid;
/* !!! The following should get moved into SAL. */
if (is_shub2()) {
xpc_sh2_IPI_access0_sn2 =
(u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0));
xpc_sh2_IPI_access1_sn2 =
(u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1));
xpc_sh2_IPI_access2_sn2 =
(u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2));
xpc_sh2_IPI_access3_sn2 =
(u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3));
for_each_online_node(node) {
nasid = cnodeid_to_nasid(node);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
-1UL);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
-1UL);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
-1UL);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
-1UL);
}
} else {
xpc_sh1_IPI_access_sn2 =
(u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS));
for_each_online_node(node) {
nasid = cnodeid_to_nasid(node);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
-1UL);
}
}
}
/*
* Restrict protections to disallow IPI operations.
*/
static void
xpc_disallow_IPI_ops_sn2(void)
{
int node;
int nasid;
/* !!! The following should get moved into SAL. */
if (is_shub2()) {
for_each_online_node(node) {
nasid = cnodeid_to_nasid(node);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
xpc_sh2_IPI_access0_sn2);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
xpc_sh2_IPI_access1_sn2);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
xpc_sh2_IPI_access2_sn2);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
xpc_sh2_IPI_access3_sn2);
}
} else {
for_each_online_node(node) {
nasid = cnodeid_to_nasid(node);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
xpc_sh1_IPI_access_sn2);
}
}
}
/*
* The following set of functions are used for the sending and receiving of
* IRQs (also known as IPIs). There are two flavors of IRQs, one that is
* associated with partition activity (SGI_XPC_ACTIVATE) and the other that
* is associated with channel activity (SGI_XPC_NOTIFY).
*/
static u64
xpc_receive_IRQ_amo_sn2(struct amo *amo)
{
return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR);
}
static enum xp_retval
xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid,
int vector)
{
int ret = 0;
unsigned long irq_flags;
local_irq_save(irq_flags);
FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag);
sn_send_IPI_phys(nasid, phys_cpuid, vector, 0);
/*
* We must always use the nofault function regardless of whether we
* are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
* didn't, we'd never know that the other partition is down and would
* keep sending IRQs and amos to it until the heartbeat times out.
*/
ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable),
xp_nofault_PIOR_target));
local_irq_restore(irq_flags);
return (ret == 0) ? xpSuccess : xpPioReadError;
}
static struct amo *
xpc_init_IRQ_amo_sn2(int index)
{
struct amo *amo = xpc_vars_sn2->amos_page + index;
(void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */
return amo;
}
/*
* Functions associated with SGI_XPC_ACTIVATE IRQ.
*/
/*
* Notify the heartbeat check thread that an activate IRQ has been received.
*/
static irqreturn_t
xpc_handle_activate_IRQ_sn2(int irq, void *dev_id)
{
unsigned long irq_flags;
spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
xpc_activate_IRQ_rcvd++;
spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
wake_up_interruptible(&xpc_activate_IRQ_wq);
return IRQ_HANDLED;
}
/*
* Flag the appropriate amo variable and send an IRQ to the specified node.
*/
static void
xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid,
int to_nasid, int to_phys_cpuid)
{
struct amo *amos = (struct amo *)__va(amos_page_pa +
(XPC_ACTIVATE_IRQ_AMOS_SN2 *
sizeof(struct amo)));
(void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)],
BIT_MASK(from_nasid / 2), to_nasid,
to_phys_cpuid, SGI_XPC_ACTIVATE);
}
static void
xpc_send_local_activate_IRQ_sn2(int from_nasid)
{
unsigned long irq_flags;
struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa +
(XPC_ACTIVATE_IRQ_AMOS_SN2 *
sizeof(struct amo)));
/* fake the sending and receipt of an activate IRQ from remote nasid */
FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable),
FETCHOP_OR, BIT_MASK(from_nasid / 2));
spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
xpc_activate_IRQ_rcvd++;
spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
wake_up_interruptible(&xpc_activate_IRQ_wq);
}
/*
* Functions associated with SGI_XPC_NOTIFY IRQ.
*/
/*
* Check to see if any chctl flags were sent from the specified partition.
*/
static void
xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part)
{
union xpc_channel_ctl_flags chctl;
unsigned long irq_flags;
chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2.
local_chctl_amo_va);
if (chctl.all_flags == 0)
return;
spin_lock_irqsave(&part->chctl_lock, irq_flags);
part->chctl.all_flags |= chctl.all_flags;
spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags="
"0x%llx\n", XPC_PARTID(part), chctl.all_flags);
xpc_wakeup_channel_mgr(part);
}
/*
* Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified
* partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more
* than one partition, we use an amo structure per partition to indicate
* whether a partition has sent an IRQ or not. If it has, then wake up the
* associated kthread to handle it.
*
* All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC
* running on other partitions.
*
* Noteworthy Arguments:
*
* irq - Interrupt ReQuest number. NOT USED.
*
* dev_id - partid of IRQ's potential sender.
*/
static irqreturn_t
xpc_handle_notify_IRQ_sn2(int irq, void *dev_id)
{
short partid = (short)(u64)dev_id;
struct xpc_partition *part = &xpc_partitions[partid];
DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2);
if (xpc_part_ref(part)) {
xpc_check_for_sent_chctl_flags_sn2(part);
xpc_part_deref(part);
}
return IRQ_HANDLED;
}
/*
* Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor
* because the write to their associated amo variable completed after the IRQ
* was received.
*/
static void
xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part)
{
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
if (xpc_part_ref(part)) {
xpc_check_for_sent_chctl_flags_sn2(part);
part_sn2->dropped_notify_IRQ_timer.expires = jiffies +
XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
add_timer(&part_sn2->dropped_notify_IRQ_timer);
xpc_part_deref(part);
}
}
/*
* Send a notify IRQ to the remote partition that is associated with the
* specified channel.
*/
static void
xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
char *chctl_flag_string, unsigned long *irq_flags)
{
struct xpc_partition *part = &xpc_partitions[ch->partid];
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
union xpc_channel_ctl_flags chctl = { 0 };
enum xp_retval ret;
if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) {
chctl.flags[ch->number] = chctl_flag;
ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va,
chctl.all_flags,
part_sn2->notify_IRQ_nasid,
part_sn2->notify_IRQ_phys_cpuid,
SGI_XPC_NOTIFY);
dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n",
chctl_flag_string, ch->partid, ch->number, ret);
if (unlikely(ret != xpSuccess)) {
if (irq_flags != NULL)
spin_unlock_irqrestore(&ch->lock, *irq_flags);
XPC_DEACTIVATE_PARTITION(part, ret);
if (irq_flags != NULL)
spin_lock_irqsave(&ch->lock, *irq_flags);
}
}
}
#define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \
xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f)
/*
* Make it look like the remote partition, which is associated with the
* specified channel, sent us a notify IRQ. This faked IRQ will be handled
* by xpc_check_for_dropped_notify_IRQ_sn2().
*/
static void
xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
char *chctl_flag_string)
{
struct xpc_partition *part = &xpc_partitions[ch->partid];
union xpc_channel_ctl_flags chctl = { 0 };
chctl.flags[ch->number] = chctl_flag;
FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va->
variable), FETCHOP_OR, chctl.all_flags);
dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n",
chctl_flag_string, ch->partid, ch->number);
}
#define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \
xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f)
static void
xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch,
unsigned long *irq_flags)
{
struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
args->reason = ch->reason;
XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags);
}
static void
xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
{
XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags);
}
static void
xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
{
struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
args->entry_size = ch->entry_size;
args->local_nentries = ch->local_nentries;
XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags);
}
static void
xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
{
struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
args->remote_nentries = ch->remote_nentries;
args->local_nentries = ch->local_nentries;
args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue);
XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags);
}
static void
xpc_send_chctl_opencomplete_sn2(struct xpc_channel *ch,
unsigned long *irq_flags)
{
XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENCOMPLETE, irq_flags);
}
static void
xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch)
{
XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL);
}
static void
xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch)
{
XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST);
}
static enum xp_retval
xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch,
unsigned long msgqueue_pa)
{
ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa;
return xpSuccess;
}
/*
* This next set of functions are used to keep track of when a partition is
* potentially engaged in accessing memory belonging to another partition.
*/
static void
xpc_indicate_partition_engaged_sn2(struct xpc_partition *part)
{
unsigned long irq_flags;
struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
(XPC_ENGAGED_PARTITIONS_AMO_SN2 *
sizeof(struct amo)));
local_irq_save(irq_flags);
/* set bit corresponding to our partid in remote partition's amo */
FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
BIT(sn_partition_id));
/*
* We must always use the nofault function regardless of whether we
* are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
* didn't, we'd never know that the other partition is down and would
* keep sending IRQs and amos to it until the heartbeat times out.
*/
(void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
variable),
xp_nofault_PIOR_target));
local_irq_restore(irq_flags);
}
static void
xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
{
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
unsigned long irq_flags;
struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
(XPC_ENGAGED_PARTITIONS_AMO_SN2 *
sizeof(struct amo)));
local_irq_save(irq_flags);
/* clear bit corresponding to our partid in remote partition's amo */
FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
~BIT(sn_partition_id));
/*
* We must always use the nofault function regardless of whether we
* are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
* didn't, we'd never know that the other partition is down and would
* keep sending IRQs and amos to it until the heartbeat times out.
*/
(void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
variable),
xp_nofault_PIOR_target));
local_irq_restore(irq_flags);
/*
* Send activate IRQ to get other side to see that we've cleared our
* bit in their engaged partitions amo.
*/
xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
cnodeid_to_nasid(0),
part_sn2->activate_IRQ_nasid,
part_sn2->activate_IRQ_phys_cpuid);
}
static void
xpc_assume_partition_disengaged_sn2(short partid)
{
struct amo *amo = xpc_vars_sn2->amos_page +
XPC_ENGAGED_PARTITIONS_AMO_SN2;
/* clear bit(s) based on partid mask in our partition's amo */
FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
~BIT(partid));
}
static int
xpc_partition_engaged_sn2(short partid)
{
struct amo *amo = xpc_vars_sn2->amos_page +
XPC_ENGAGED_PARTITIONS_AMO_SN2;
/* our partition's amo variable ANDed with partid mask */
return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
BIT(partid)) != 0;
}
static int
xpc_any_partition_engaged_sn2(void)
{
struct amo *amo = xpc_vars_sn2->amos_page +
XPC_ENGAGED_PARTITIONS_AMO_SN2;
/* our partition's amo variable */
return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0;
}
/* original protection values for each node */
static u64 xpc_prot_vec_sn2[MAX_NUMNODES];
/*
* Change protections to allow amo operations on non-Shub 1.1 systems.
*/
static enum xp_retval
xpc_allow_amo_ops_sn2(struct amo *amos_page)
{
enum xp_retval ret = xpSuccess;
/*
* On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST
* collides with memory operations. On those systems we call
* xpc_allow_amo_ops_shub_wars_1_1_sn2() instead.
*/
if (!enable_shub_wars_1_1())
ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE);
return ret;
}
/*
* Change protections to allow amo operations on Shub 1.1 systems.
*/
static void
xpc_allow_amo_ops_shub_wars_1_1_sn2(void)
{
int node;
int nasid;
if (!enable_shub_wars_1_1())
return;
for_each_online_node(node) {
nasid = cnodeid_to_nasid(node);
/* save current protection values */
xpc_prot_vec_sn2[node] =
(u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid,
SH1_MD_DQLP_MMR_DIR_PRIVEC0));
/* open up everything */
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
SH1_MD_DQLP_MMR_DIR_PRIVEC0),
-1UL);
HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
SH1_MD_DQRP_MMR_DIR_PRIVEC0),
-1UL);
}
}
static enum xp_retval
xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa,
size_t *len)
{
s64 status;
enum xp_retval ret;
status = sn_partition_reserved_page_pa((u64)buf, cookie,
(u64 *)rp_pa, (u64 *)len);
if (status == SALRET_OK)
ret = xpSuccess;
else if (status == SALRET_MORE_PASSES)
ret = xpNeedMoreInfo;
else
ret = xpSalError;
return ret;
}
static int
xpc_setup_rsvd_page_sn2(struct xpc_rsvd_page *rp)
{
struct amo *amos_page;
int i;
int ret;
xpc_vars_sn2 = XPC_RP_VARS(rp);
rp->sn.sn2.vars_pa = xp_pa(xpc_vars_sn2);
/* vars_part array follows immediately after vars */
xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) +
XPC_RP_VARS_SIZE);
/*
* Before clearing xpc_vars_sn2, see if a page of amos had been
* previously allocated. If not we'll need to allocate one and set
* permissions so that cross-partition amos are allowed.
*
* The allocated amo page needs MCA reporting to remain disabled after
* XPC has unloaded. To make this work, we keep a copy of the pointer
* to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure,
* which is pointed to by the reserved page, and re-use that saved copy
* on subsequent loads of XPC. This amo page is never freed, and its
* memory protections are never restricted.
*/
amos_page = xpc_vars_sn2->amos_page;
if (amos_page == NULL) {
amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1));
if (amos_page == NULL) {
dev_err(xpc_part, "can't allocate page of amos\n");
return -ENOMEM;
}
/*
* Open up amo-R/W to cpu. This is done on Shub 1.1 systems
* when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called.
*/
ret = xpc_allow_amo_ops_sn2(amos_page);
if (ret != xpSuccess) {
dev_err(xpc_part, "can't allow amo operations\n");
uncached_free_page(__IA64_UNCACHED_OFFSET |
TO_PHYS((u64)amos_page), 1);
return -EPERM;
}
}
/* clear xpc_vars_sn2 */
memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2));
xpc_vars_sn2->version = XPC_V_VERSION;
xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0);
xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0);
xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2);
xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page);
xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */
/* clear xpc_vars_part_sn2 */
memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) *
XP_MAX_NPARTITIONS_SN2);
/* initialize the activate IRQ related amo variables */
for (i = 0; i < xpc_nasid_mask_nlongs; i++)
(void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i);
/* initialize the engaged remote partitions related amo variables */
(void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2);
(void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2);
return 0;
}
static int
xpc_hb_allowed_sn2(short partid, void *heartbeating_to_mask)
{
return test_bit(partid, heartbeating_to_mask);
}
static void
xpc_allow_hb_sn2(short partid)
{
DBUG_ON(xpc_vars_sn2 == NULL);
set_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
}
static void
xpc_disallow_hb_sn2(short partid)
{
DBUG_ON(xpc_vars_sn2 == NULL);
clear_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
}
static void
xpc_disallow_all_hbs_sn2(void)
{
DBUG_ON(xpc_vars_sn2 == NULL);
bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, xp_max_npartitions);
}
static void
xpc_increment_heartbeat_sn2(void)
{
xpc_vars_sn2->heartbeat++;
}
static void
xpc_offline_heartbeat_sn2(void)
{
xpc_increment_heartbeat_sn2();
xpc_vars_sn2->heartbeat_offline = 1;
}
static void
xpc_online_heartbeat_sn2(void)
{
xpc_increment_heartbeat_sn2();
xpc_vars_sn2->heartbeat_offline = 0;
}
static void
xpc_heartbeat_init_sn2(void)
{
DBUG_ON(xpc_vars_sn2 == NULL);
bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2);
xpc_online_heartbeat_sn2();
}
static void
xpc_heartbeat_exit_sn2(void)
{
xpc_offline_heartbeat_sn2();
}
static enum xp_retval
xpc_get_remote_heartbeat_sn2(struct xpc_partition *part)
{
struct xpc_vars_sn2 *remote_vars;
enum xp_retval ret;
remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
/* pull the remote vars structure that contains the heartbeat */
ret = xp_remote_memcpy(xp_pa(remote_vars),
part->sn.sn2.remote_vars_pa,
XPC_RP_VARS_SIZE);
if (ret != xpSuccess)
return ret;
dev_dbg(xpc_part, "partid=%d, heartbeat=%lld, last_heartbeat=%lld, "
"heartbeat_offline=%lld, HB_mask[0]=0x%lx\n", XPC_PARTID(part),
remote_vars->heartbeat, part->last_heartbeat,
remote_vars->heartbeat_offline,
remote_vars->heartbeating_to_mask[0]);
if ((remote_vars->heartbeat == part->last_heartbeat &&
!remote_vars->heartbeat_offline) ||
!xpc_hb_allowed_sn2(sn_partition_id,
remote_vars->heartbeating_to_mask)) {
ret = xpNoHeartbeat;
} else {
part->last_heartbeat = remote_vars->heartbeat;
}
return ret;
}
/*
* Get a copy of the remote partition's XPC variables from the reserved page.
*
* remote_vars points to a buffer that is cacheline aligned for BTE copies and
* assumed to be of size XPC_RP_VARS_SIZE.
*/
static enum xp_retval
xpc_get_remote_vars_sn2(unsigned long remote_vars_pa,
struct xpc_vars_sn2 *remote_vars)
{
enum xp_retval ret;
if (remote_vars_pa == 0)
return xpVarsNotSet;
/* pull over the cross partition variables */
ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa,
XPC_RP_VARS_SIZE);
if (ret != xpSuccess)
return ret;
if (XPC_VERSION_MAJOR(remote_vars->version) !=
XPC_VERSION_MAJOR(XPC_V_VERSION)) {
return xpBadVersion;
}
return xpSuccess;
}
static void
xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp,
unsigned long remote_rp_pa, int nasid)
{
xpc_send_local_activate_IRQ_sn2(nasid);
}
static void
xpc_request_partition_reactivation_sn2(struct xpc_partition *part)
{
xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid);
}
static void
xpc_request_partition_deactivation_sn2(struct xpc_partition *part)
{
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
unsigned long irq_flags;
struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
(XPC_DEACTIVATE_REQUEST_AMO_SN2 *
sizeof(struct amo)));
local_irq_save(irq_flags);
/* set bit corresponding to our partid in remote partition's amo */
FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
BIT(sn_partition_id));
/*
* We must always use the nofault function regardless of whether we
* are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
* didn't, we'd never know that the other partition is down and would
* keep sending IRQs and amos to it until the heartbeat times out.
*/
(void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
variable),
xp_nofault_PIOR_target));
local_irq_restore(irq_flags);
/*
* Send activate IRQ to get other side to see that we've set our
* bit in their deactivate request amo.
*/
xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
cnodeid_to_nasid(0),
part_sn2->activate_IRQ_nasid,
part_sn2->activate_IRQ_phys_cpuid);
}
static void
xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
{
unsigned long irq_flags;
struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
(XPC_DEACTIVATE_REQUEST_AMO_SN2 *
sizeof(struct amo)));
local_irq_save(irq_flags);
/* clear bit corresponding to our partid in remote partition's amo */
FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
~BIT(sn_partition_id));
/*
* We must always use the nofault function regardless of whether we
* are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
* didn't, we'd never know that the other partition is down and would
* keep sending IRQs and amos to it until the heartbeat times out.
*/
(void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
variable),
xp_nofault_PIOR_target));
local_irq_restore(irq_flags);
}
static int
xpc_partition_deactivation_requested_sn2(short partid)
{
struct amo *amo = xpc_vars_sn2->amos_page +
XPC_DEACTIVATE_REQUEST_AMO_SN2;
/* our partition's amo variable ANDed with partid mask */
return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
BIT(partid)) != 0;
}
/*
* Update the remote partition's info.
*/
static void
xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version,
unsigned long *remote_rp_ts_jiffies,
unsigned long remote_rp_pa,
unsigned long remote_vars_pa,
struct xpc_vars_sn2 *remote_vars)
{
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
part->remote_rp_version = remote_rp_version;
dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n",
part->remote_rp_version);
part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies;
dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n",
part->remote_rp_ts_jiffies);
part->remote_rp_pa = remote_rp_pa;
dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa);
part_sn2->remote_vars_pa = remote_vars_pa;
dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n",
part_sn2->remote_vars_pa);
part->last_heartbeat = remote_vars->heartbeat - 1;
dev_dbg(xpc_part, " last_heartbeat = 0x%016llx\n",
part->last_heartbeat);
part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa;
dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n",
part_sn2->remote_vars_part_pa);
part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid;
dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n",
part_sn2->activate_IRQ_nasid);
part_sn2->activate_IRQ_phys_cpuid =
remote_vars->activate_IRQ_phys_cpuid;
dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n",
part_sn2->activate_IRQ_phys_cpuid);
part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa;
dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n",
part_sn2->remote_amos_page_pa);
part_sn2->remote_vars_version = remote_vars->version;
dev_dbg(xpc_part, " remote_vars_version = 0x%x\n",
part_sn2->remote_vars_version);
}
/*
* Prior code has determined the nasid which generated a activate IRQ.
* Inspect that nasid to determine if its partition needs to be activated
* or deactivated.
*
* A partition is considered "awaiting activation" if our partition
* flags indicate it is not active and it has a heartbeat. A
* partition is considered "awaiting deactivation" if our partition
* flags indicate it is active but it has no heartbeat or it is not
* sending its heartbeat to us.
*
* To determine the heartbeat, the remote nasid must have a properly
* initialized reserved page.
*/
static void
xpc_identify_activate_IRQ_req_sn2(int nasid)
{
struct xpc_rsvd_page *remote_rp;
struct xpc_vars_sn2 *remote_vars;
unsigned long remote_rp_pa;
unsigned long remote_vars_pa;
int remote_rp_version;
int reactivate = 0;
unsigned long remote_rp_ts_jiffies = 0;
short partid;
struct xpc_partition *part;
struct xpc_partition_sn2 *part_sn2;
enum xp_retval ret;
/* pull over the reserved page structure */
remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2;
ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa);
if (ret != xpSuccess) {
dev_warn(xpc_part, "unable to get reserved page from nasid %d, "
"which sent interrupt, reason=%d\n", nasid, ret);
return;
}
remote_vars_pa = remote_rp->sn.sn2.vars_pa;
remote_rp_version = remote_rp->version;
remote_rp_ts_jiffies = remote_rp->ts_jiffies;
partid = remote_rp->SAL_partid;
part = &xpc_partitions[partid];
part_sn2 = &part->sn.sn2;
/* pull over the cross partition variables */
remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars);
if (ret != xpSuccess) {
dev_warn(xpc_part, "unable to get XPC variables from nasid %d, "
"which sent interrupt, reason=%d\n", nasid, ret);
XPC_DEACTIVATE_PARTITION(part, ret);
return;
}
part->activate_IRQ_rcvd++;
dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = "
"%lld:0x%lx\n", (int)nasid, (int)partid,
part->activate_IRQ_rcvd,
remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]);
if (xpc_partition_disengaged(part) &&
part->act_state == XPC_P_AS_INACTIVE) {
xpc_update_partition_info_sn2(part, remote_rp_version,
&remote_rp_ts_jiffies,
remote_rp_pa, remote_vars_pa,
remote_vars);
if (xpc_partition_deactivation_requested_sn2(partid)) {
/*
* Other side is waiting on us to deactivate even though
* we already have.
*/
return;
}
xpc_activate_partition(part);
return;
}
DBUG_ON(part->remote_rp_version == 0);
DBUG_ON(part_sn2->remote_vars_version == 0);
if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) {
/* the other side rebooted */
DBUG_ON(xpc_partition_engaged_sn2(partid));
DBUG_ON(xpc_partition_deactivation_requested_sn2(partid));
xpc_update_partition_info_sn2(part, remote_rp_version,
&remote_rp_ts_jiffies,
remote_rp_pa, remote_vars_pa,
remote_vars);
reactivate = 1;
}
if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) {
/* still waiting on other side to disengage from us */
return;
}
if (reactivate)
XPC_DEACTIVATE_PARTITION(part, xpReactivating);
else if (xpc_partition_deactivation_requested_sn2(partid))
XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown);
}
/*
* Loop through the activation amo variables and process any bits
* which are set. Each bit indicates a nasid sending a partition
* activation or deactivation request.
*
* Return #of IRQs detected.
*/
int
xpc_identify_activate_IRQ_sender_sn2(void)
{
int l;
int b;
unsigned long nasid_mask_long;
u64 nasid; /* remote nasid */
int n_IRQs_detected = 0;
struct amo *act_amos;
act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2;
/* scan through activate amo variables looking for non-zero entries */
for (l = 0; l < xpc_nasid_mask_nlongs; l++) {
if (xpc_exiting)
break;
nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]);
b = find_first_bit(&nasid_mask_long, BITS_PER_LONG);
if (b >= BITS_PER_LONG) {
/* no IRQs from nasids in this amo variable */
continue;
}
dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l,
nasid_mask_long);
/*
* If this nasid has been added to the machine since
* our partition was reset, this will retain the
* remote nasid in our reserved pages machine mask.
* This is used in the event of module reload.
*/
xpc_mach_nasids[l] |= nasid_mask_long;
/* locate the nasid(s) which sent interrupts */
do {
n_IRQs_detected++;
nasid = (l * BITS_PER_LONG + b) * 2;
dev_dbg(xpc_part, "interrupt from nasid %lld\n", nasid);
xpc_identify_activate_IRQ_req_sn2(nasid);
b = find_next_bit(&nasid_mask_long, BITS_PER_LONG,
b + 1);
} while (b < BITS_PER_LONG);
}
return n_IRQs_detected;
}
static void
xpc_process_activate_IRQ_rcvd_sn2(void)
{
unsigned long irq_flags;
int n_IRQs_expected;
int n_IRQs_detected;
spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
n_IRQs_expected = xpc_activate_IRQ_rcvd;
xpc_activate_IRQ_rcvd = 0;
spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2();
if (n_IRQs_detected < n_IRQs_expected) {
/* retry once to help avoid missing amo */
(void)xpc_identify_activate_IRQ_sender_sn2();
}
}
/*
* Setup the channel structures that are sn2 specific.
*/
static enum xp_retval
xpc_setup_ch_structures_sn2(struct xpc_partition *part)
{
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
struct xpc_channel_sn2 *ch_sn2;
enum xp_retval retval;
int ret;
int cpuid;
int ch_number;
struct timer_list *timer;
short partid = XPC_PARTID(part);
/* allocate all the required GET/PUT values */
part_sn2->local_GPs =
xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
&part_sn2->local_GPs_base);
if (part_sn2->local_GPs == NULL) {
dev_err(xpc_chan, "can't get memory for local get/put "
"values\n");
return xpNoMemory;
}
part_sn2->remote_GPs =
xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
&part_sn2->remote_GPs_base);
if (part_sn2->remote_GPs == NULL) {
dev_err(xpc_chan, "can't get memory for remote get/put "
"values\n");
retval = xpNoMemory;
goto out_1;
}
part_sn2->remote_GPs_pa = 0;
/* allocate all the required open and close args */
part_sn2->local_openclose_args =
xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE,
GFP_KERNEL, &part_sn2->
local_openclose_args_base);
if (part_sn2->local_openclose_args == NULL) {
dev_err(xpc_chan, "can't get memory for local connect args\n");
retval = xpNoMemory;
goto out_2;
}
part_sn2->remote_openclose_args_pa = 0;
part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid);
part_sn2->notify_IRQ_nasid = 0;
part_sn2->notify_IRQ_phys_cpuid = 0;
part_sn2->remote_chctl_amo_va = NULL;
sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid);
ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2,
IRQF_SHARED, part_sn2->notify_IRQ_owner,
(void *)(u64)partid);
if (ret != 0) {
dev_err(xpc_chan, "can't register NOTIFY IRQ handler, "
"errno=%d\n", -ret);
retval = xpLackOfResources;
goto out_3;
}
/* Setup a timer to check for dropped notify IRQs */
timer = &part_sn2->dropped_notify_IRQ_timer;
init_timer(timer);
timer->function =
(void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2;
timer->data = (unsigned long)part;
timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
add_timer(timer);
for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
ch_sn2 = &part->channels[ch_number].sn.sn2;
ch_sn2->local_GP = &part_sn2->local_GPs[ch_number];
ch_sn2->local_openclose_args =
&part_sn2->local_openclose_args[ch_number];
mutex_init(&ch_sn2->msg_to_pull_mutex);
}
/*
* Setup the per partition specific variables required by the
* remote partition to establish channel connections with us.
*
* The setting of the magic # indicates that these per partition
* specific variables are ready to be used.
*/
xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs);
xpc_vars_part_sn2[partid].openclose_args_pa =
xp_pa(part_sn2->local_openclose_args);
xpc_vars_part_sn2[partid].chctl_amo_pa =
xp_pa(part_sn2->local_chctl_amo_va);
cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */
xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid);
xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid =
cpu_physical_id(cpuid);
xpc_vars_part_sn2[partid].nchannels = part->nchannels;
xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2;
return xpSuccess;
/* setup of ch structures failed */
out_3:
kfree(part_sn2->local_openclose_args_base);
part_sn2->local_openclose_args = NULL;
out_2:
kfree(part_sn2->remote_GPs_base);
part_sn2->remote_GPs = NULL;
out_1:
kfree(part_sn2->local_GPs_base);
part_sn2->local_GPs = NULL;
return retval;
}
/*
* Teardown the channel structures that are sn2 specific.
*/
static void
xpc_teardown_ch_structures_sn2(struct xpc_partition *part)
{
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
short partid = XPC_PARTID(part);
/*
* Indicate that the variables specific to the remote partition are no
* longer available for its use.
*/
xpc_vars_part_sn2[partid].magic = 0;
/* in case we've still got outstanding timers registered... */
del_timer_sync(&part_sn2->dropped_notify_IRQ_timer);
free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid);
kfree(part_sn2->local_openclose_args_base);
part_sn2->local_openclose_args = NULL;
kfree(part_sn2->remote_GPs_base);
part_sn2->remote_GPs = NULL;
kfree(part_sn2->local_GPs_base);
part_sn2->local_GPs = NULL;
part_sn2->local_chctl_amo_va = NULL;
}
/*
* Create a wrapper that hides the underlying mechanism for pulling a cacheline
* (or multiple cachelines) from a remote partition.
*
* src_pa must be a cacheline aligned physical address on the remote partition.
* dst must be a cacheline aligned virtual address on this partition.
* cnt must be cacheline sized
*/
/* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */
static enum xp_retval
xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst,
const unsigned long src_pa, size_t cnt)
{
enum xp_retval ret;
DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa));
DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst));
DBUG_ON(cnt != L1_CACHE_ALIGN(cnt));
if (part->act_state == XPC_P_AS_DEACTIVATING)
return part->reason;
ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt);
if (ret != xpSuccess) {
dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed,"
" ret=%d\n", XPC_PARTID(part), ret);
}
return ret;
}
/*
* Pull the remote per partition specific variables from the specified
* partition.
*/
static enum xp_retval
xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
{
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
u8 buffer[L1_CACHE_BYTES * 2];
struct xpc_vars_part_sn2 *pulled_entry_cacheline =
(struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer);
struct xpc_vars_part_sn2 *pulled_entry;
unsigned long remote_entry_cacheline_pa;
unsigned long remote_entry_pa;
short partid = XPC_PARTID(part);
enum xp_retval ret;
/* pull the cacheline that contains the variables we're interested in */
DBUG_ON(part_sn2->remote_vars_part_pa !=
L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa));
DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2);
remote_entry_pa = part_sn2->remote_vars_part_pa +
sn_partition_id * sizeof(struct xpc_vars_part_sn2);
remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1));
pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline
+ (remote_entry_pa &
(L1_CACHE_BYTES - 1)));
ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline,
remote_entry_cacheline_pa,
L1_CACHE_BYTES);
if (ret != xpSuccess) {
dev_dbg(xpc_chan, "failed to pull XPC vars_part from "
"partition %d, ret=%d\n", partid, ret);
return ret;
}
/* see if they've been set up yet */
if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 &&
pulled_entry->magic != XPC_VP_MAGIC2_SN2) {
if (pulled_entry->magic != 0) {
dev_dbg(xpc_chan, "partition %d's XPC vars_part for "
"partition %d has bad magic value (=0x%llx)\n",
partid, sn_partition_id, pulled_entry->magic);
return xpBadMagic;
}
/* they've not been initialized yet */
return xpRetry;
}
if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) {
/* validate the variables */
if (pulled_entry->GPs_pa == 0 ||
pulled_entry->openclose_args_pa == 0 ||
pulled_entry->chctl_amo_pa == 0) {
dev_err(xpc_chan, "partition %d's XPC vars_part for "
"partition %d are not valid\n", partid,
sn_partition_id);
return xpInvalidAddress;
}
/* the variables we imported look to be valid */
part_sn2->remote_GPs_pa = pulled_entry->GPs_pa;
part_sn2->remote_openclose_args_pa =
pulled_entry->openclose_args_pa;
part_sn2->remote_chctl_amo_va =
(struct amo *)__va(pulled_entry->chctl_amo_pa);
part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid;
part_sn2->notify_IRQ_phys_cpuid =
pulled_entry->notify_IRQ_phys_cpuid;
if (part->nchannels > pulled_entry->nchannels)
part->nchannels = pulled_entry->nchannels;
/* let the other side know that we've pulled their variables */
xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2;
}
if (pulled_entry->magic == XPC_VP_MAGIC1_SN2)
return xpRetry;
return xpSuccess;
}
/*
* Establish first contact with the remote partititon. This involves pulling
* the XPC per partition variables from the remote partition and waiting for
* the remote partition to pull ours.
*/
static enum xp_retval
xpc_make_first_contact_sn2(struct xpc_partition *part)
{
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
enum xp_retval ret;
/*
* Register the remote partition's amos with SAL so it can handle
* and cleanup errors within that address range should the remote
* partition go down. We don't unregister this range because it is
* difficult to tell when outstanding writes to the remote partition
* are finished and thus when it is safe to unregister. This should
* not result in wasted space in the SAL xp_addr_region table because
* we should get the same page for remote_amos_page_pa after module
* reloads and system reboots.
*/
if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa,
PAGE_SIZE, 1) < 0) {
dev_warn(xpc_part, "xpc_activating(%d) failed to register "
"xp_addr region\n", XPC_PARTID(part));
ret = xpPhysAddrRegFailed;
XPC_DEACTIVATE_PARTITION(part, ret);
return ret;
}
/*
* Send activate IRQ to get other side to activate if they've not
* already begun to do so.
*/
xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
cnodeid_to_nasid(0),
part_sn2->activate_IRQ_nasid,
part_sn2->activate_IRQ_phys_cpuid);
while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) {
if (ret != xpRetry) {
XPC_DEACTIVATE_PARTITION(part, ret);
return ret;
}
dev_dbg(xpc_part, "waiting to make first contact with "
"partition %d\n", XPC_PARTID(part));
/* wait a 1/4 of a second or so */
(void)msleep_interruptible(250);
if (part->act_state == XPC_P_AS_DEACTIVATING)
return part->reason;
}
return xpSuccess;
}
/*
* Get the chctl flags and pull the openclose args and/or remote GPs as needed.
*/
static u64
xpc_get_chctl_all_flags_sn2(struct xpc_partition *part)
{
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
unsigned long irq_flags;
union xpc_channel_ctl_flags chctl;
enum xp_retval ret;
/*
* See if there are any chctl flags to be handled.
*/
spin_lock_irqsave(&part->chctl_lock, irq_flags);
chctl = part->chctl;
if (chctl.all_flags != 0)
part->chctl.all_flags = 0;
spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
if (xpc_any_openclose_chctl_flags_set(&chctl)) {
ret = xpc_pull_remote_cachelines_sn2(part, part->
remote_openclose_args,
part_sn2->
remote_openclose_args_pa,
XPC_OPENCLOSE_ARGS_SIZE);
if (ret != xpSuccess) {
XPC_DEACTIVATE_PARTITION(part, ret);
dev_dbg(xpc_chan, "failed to pull openclose args from "
"partition %d, ret=%d\n", XPC_PARTID(part),
ret);
/* don't bother processing chctl flags anymore */
chctl.all_flags = 0;
}
}
if (xpc_any_msg_chctl_flags_set(&chctl)) {
ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs,
part_sn2->remote_GPs_pa,
XPC_GP_SIZE);
if (ret != xpSuccess) {
XPC_DEACTIVATE_PARTITION(part, ret);
dev_dbg(xpc_chan, "failed to pull GPs from partition "
"%d, ret=%d\n", XPC_PARTID(part), ret);
/* don't bother processing chctl flags anymore */
chctl.all_flags = 0;
}
}
return chctl.all_flags;
}
/*
* Allocate the local message queue and the notify queue.
*/
static enum xp_retval
xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
unsigned long irq_flags;
int nentries;
size_t nbytes;
for (nentries = ch->local_nentries; nentries > 0; nentries--) {
nbytes = nentries * ch->entry_size;
ch_sn2->local_msgqueue =
xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL,
&ch_sn2->local_msgqueue_base);
if (ch_sn2->local_msgqueue == NULL)
continue;
nbytes = nentries * sizeof(struct xpc_notify_sn2);
ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL);
if (ch_sn2->notify_queue == NULL) {
kfree(ch_sn2->local_msgqueue_base);
ch_sn2->local_msgqueue = NULL;
continue;
}
spin_lock_irqsave(&ch->lock, irq_flags);
if (nentries < ch->local_nentries) {
dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, "
"partid=%d, channel=%d\n", nentries,
ch->local_nentries, ch->partid, ch->number);
ch->local_nentries = nentries;
}
spin_unlock_irqrestore(&ch->lock, irq_flags);
return xpSuccess;
}
dev_dbg(xpc_chan, "can't get memory for local message queue and notify "
"queue, partid=%d, channel=%d\n", ch->partid, ch->number);
return xpNoMemory;
}
/*
* Allocate the cached remote message queue.
*/
static enum xp_retval
xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
unsigned long irq_flags;
int nentries;
size_t nbytes;
DBUG_ON(ch->remote_nentries <= 0);
for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
nbytes = nentries * ch->entry_size;
ch_sn2->remote_msgqueue =
xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2->
remote_msgqueue_base);
if (ch_sn2->remote_msgqueue == NULL)
continue;
spin_lock_irqsave(&ch->lock, irq_flags);
if (nentries < ch->remote_nentries) {
dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, "
"partid=%d, channel=%d\n", nentries,
ch->remote_nentries, ch->partid, ch->number);
ch->remote_nentries = nentries;
}
spin_unlock_irqrestore(&ch->lock, irq_flags);
return xpSuccess;
}
dev_dbg(xpc_chan, "can't get memory for cached remote message queue, "
"partid=%d, channel=%d\n", ch->partid, ch->number);
return xpNoMemory;
}
/*
* Allocate message queues and other stuff associated with a channel.
*
* Note: Assumes all of the channel sizes are filled in.
*/
static enum xp_retval
xpc_setup_msg_structures_sn2(struct xpc_channel *ch)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
enum xp_retval ret;
DBUG_ON(ch->flags & XPC_C_SETUP);
ret = xpc_allocate_local_msgqueue_sn2(ch);
if (ret == xpSuccess) {
ret = xpc_allocate_remote_msgqueue_sn2(ch);
if (ret != xpSuccess) {
kfree(ch_sn2->local_msgqueue_base);
ch_sn2->local_msgqueue = NULL;
kfree(ch_sn2->notify_queue);
ch_sn2->notify_queue = NULL;
}
}
return ret;
}
/*
* Free up message queues and other stuff that were allocated for the specified
* channel.
*/
static void
xpc_teardown_msg_structures_sn2(struct xpc_channel *ch)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
DBUG_ON(!spin_is_locked(&ch->lock));
ch_sn2->remote_msgqueue_pa = 0;
ch_sn2->local_GP->get = 0;
ch_sn2->local_GP->put = 0;
ch_sn2->remote_GP.get = 0;
ch_sn2->remote_GP.put = 0;
ch_sn2->w_local_GP.get = 0;
ch_sn2->w_local_GP.put = 0;
ch_sn2->w_remote_GP.get = 0;
ch_sn2->w_remote_GP.put = 0;
ch_sn2->next_msg_to_pull = 0;
if (ch->flags & XPC_C_SETUP) {
dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n",
ch->flags, ch->partid, ch->number);
kfree(ch_sn2->local_msgqueue_base);
ch_sn2->local_msgqueue = NULL;
kfree(ch_sn2->remote_msgqueue_base);
ch_sn2->remote_msgqueue = NULL;
kfree(ch_sn2->notify_queue);
ch_sn2->notify_queue = NULL;
}
}
/*
* Notify those who wanted to be notified upon delivery of their message.
*/
static void
xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put)
{
struct xpc_notify_sn2 *notify;
u8 notify_type;
s64 get = ch->sn.sn2.w_remote_GP.get - 1;
while (++get < put && atomic_read(&ch->n_to_notify) > 0) {
notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries];
/*
* See if the notify entry indicates it was associated with
* a message who's sender wants to be notified. It is possible
* that it is, but someone else is doing or has done the
* notification.
*/
notify_type = notify->type;
if (notify_type == 0 ||
cmpxchg(&notify->type, notify_type, 0) != notify_type) {
continue;
}
DBUG_ON(notify_type != XPC_N_CALL);
atomic_dec(&ch->n_to_notify);
if (notify->func != NULL) {
dev_dbg(xpc_chan, "notify->func() called, notify=0x%p "
"msg_number=%lld partid=%d channel=%d\n",
(void *)notify, get, ch->partid, ch->number);
notify->func(reason, ch->partid, ch->number,
notify->key);
dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p"
" msg_number=%lld partid=%d channel=%d\n",
(void *)notify, get, ch->partid, ch->number);
}
}
}
static void
xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch)
{
xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put);
}
/*
* Clear some of the msg flags in the local message queue.
*/
static inline void
xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
struct xpc_msg_sn2 *msg;
s64 get;
get = ch_sn2->w_remote_GP.get;
do {
msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
(get % ch->local_nentries) *
ch->entry_size);
DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
msg->flags = 0;
} while (++get < ch_sn2->remote_GP.get);
}
/*
* Clear some of the msg flags in the remote message queue.
*/
static inline void
xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
struct xpc_msg_sn2 *msg;
s64 put, remote_nentries = ch->remote_nentries;
/* flags are zeroed when the buffer is allocated */
if (ch_sn2->remote_GP.put < remote_nentries)
return;
put = max(ch_sn2->w_remote_GP.put, remote_nentries);
do {
msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
(put % remote_nentries) *
ch->entry_size);
DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
DBUG_ON(!(msg->flags & XPC_M_SN2_DONE));
DBUG_ON(msg->number != put - remote_nentries);
msg->flags = 0;
} while (++put < ch_sn2->remote_GP.put);
}
static int
xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch)
{
return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get;
}
static void
xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number)
{
struct xpc_channel *ch = &part->channels[ch_number];
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
int npayloads_sent;
ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number];
/* See what, if anything, has changed for each connected channel */
xpc_msgqueue_ref(ch);
if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get &&
ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) {
/* nothing changed since GPs were last pulled */
xpc_msgqueue_deref(ch);
return;
}
if (!(ch->flags & XPC_C_CONNECTED)) {
xpc_msgqueue_deref(ch);
return;
}
/*
* First check to see if messages recently sent by us have been
* received by the other side. (The remote GET value will have
* changed since we last looked at it.)
*/
if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) {
/*
* We need to notify any senders that want to be notified
* that their sent messages have been received by their
* intended recipients. We need to do this before updating
* w_remote_GP.get so that we don't allocate the same message
* queue entries prematurely (see xpc_allocate_msg()).
*/
if (atomic_read(&ch->n_to_notify) > 0) {
/*
* Notify senders that messages sent have been
* received and delivered by the other side.
*/
xpc_notify_senders_sn2(ch, xpMsgDelivered,
ch_sn2->remote_GP.get);
}
/*
* Clear msg->flags in previously sent messages, so that
* they're ready for xpc_allocate_msg().
*/
xpc_clear_local_msgqueue_flags_sn2(ch);
ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get;
dev_dbg(xpc_chan, "w_remote_GP.get changed to %lld, partid=%d, "
"channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid,
ch->number);
/*
* If anyone was waiting for message queue entries to become
* available, wake them up.
*/
if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
wake_up(&ch->msg_allocate_wq);
}
/*
* Now check for newly sent messages by the other side. (The remote
* PUT value will have changed since we last looked at it.)
*/
if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) {
/*
* Clear msg->flags in previously received messages, so that
* they're ready for xpc_get_deliverable_payload_sn2().
*/
xpc_clear_remote_msgqueue_flags_sn2(ch);
smp_wmb(); /* ensure flags have been cleared before bte_copy */
ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put;
dev_dbg(xpc_chan, "w_remote_GP.put changed to %lld, partid=%d, "
"channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid,
ch->number);
npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch);
if (npayloads_sent > 0) {
dev_dbg(xpc_chan, "msgs waiting to be copied and "
"delivered=%d, partid=%d, channel=%d\n",
npayloads_sent, ch->partid, ch->number);
if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)
xpc_activate_kthreads(ch, npayloads_sent);
}
}
xpc_msgqueue_deref(ch);
}
static struct xpc_msg_sn2 *
xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
{
struct xpc_partition *part = &xpc_partitions[ch->partid];
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
unsigned long remote_msg_pa;
struct xpc_msg_sn2 *msg;
u32 msg_index;
u32 nmsgs;
u64 msg_offset;
enum xp_retval ret;
if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) {
/* we were interrupted by a signal */
return NULL;
}
while (get >= ch_sn2->next_msg_to_pull) {
/* pull as many messages as are ready and able to be pulled */
msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries;
DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put);
nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull;
if (msg_index + nmsgs > ch->remote_nentries) {
/* ignore the ones that wrap the msg queue for now */
nmsgs = ch->remote_nentries - msg_index;
}
msg_offset = msg_index * ch->entry_size;
msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
msg_offset);
remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset;
ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa,
nmsgs * ch->entry_size);
if (ret != xpSuccess) {
dev_dbg(xpc_chan, "failed to pull %d msgs starting with"
" msg %lld from partition %d, channel=%d, "
"ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull,
ch->partid, ch->number, ret);
XPC_DEACTIVATE_PARTITION(part, ret);
mutex_unlock(&ch_sn2->msg_to_pull_mutex);
return NULL;
}
ch_sn2->next_msg_to_pull += nmsgs;
}
mutex_unlock(&ch_sn2->msg_to_pull_mutex);
/* return the message we were looking for */
msg_offset = (get % ch->remote_nentries) * ch->entry_size;
msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset);
return msg;
}
/*
* Get the next deliverable message's payload.
*/
static void *
xpc_get_deliverable_payload_sn2(struct xpc_channel *ch)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
struct xpc_msg_sn2 *msg;
void *payload = NULL;
s64 get;
do {
if (ch->flags & XPC_C_DISCONNECTING)
break;
get = ch_sn2->w_local_GP.get;
smp_rmb(); /* guarantee that .get loads before .put */
if (get == ch_sn2->w_remote_GP.put)
break;
/* There are messages waiting to be pulled and delivered.
* We need to try to secure one for ourselves. We'll do this
* by trying to increment w_local_GP.get and hope that no one
* else beats us to it. If they do, we'll we'll simply have
* to try again for the next one.
*/
if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) {
/* we got the entry referenced by get */
dev_dbg(xpc_chan, "w_local_GP.get changed to %lld, "
"partid=%d, channel=%d\n", get + 1,
ch->partid, ch->number);
/* pull the message from the remote partition */
msg = xpc_pull_remote_msg_sn2(ch, get);
if (msg != NULL) {
DBUG_ON(msg->number != get);
DBUG_ON(msg->flags & XPC_M_SN2_DONE);
DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
payload = &msg->payload;
}
break;
}
} while (1);
return payload;
}
/*
* Now we actually send the messages that are ready to be sent by advancing
* the local message queue's Put value and then send a chctl msgrequest to the
* recipient partition.
*/
static void
xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
struct xpc_msg_sn2 *msg;
s64 put = initial_put + 1;
int send_msgrequest = 0;
while (1) {
while (1) {
if (put == ch_sn2->w_local_GP.put)
break;
msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
local_msgqueue + (put %
ch->local_nentries) *
ch->entry_size);
if (!(msg->flags & XPC_M_SN2_READY))
break;
put++;
}
if (put == initial_put) {
/* nothing's changed */
break;
}
if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) !=
initial_put) {
/* someone else beat us to it */
DBUG_ON(ch_sn2->local_GP->put < initial_put);
break;
}
/* we just set the new value of local_GP->put */
dev_dbg(xpc_chan, "local_GP->put changed to %lld, partid=%d, "
"channel=%d\n", put, ch->partid, ch->number);
send_msgrequest = 1;
/*
* We need to ensure that the message referenced by
* local_GP->put is not XPC_M_SN2_READY or that local_GP->put
* equals w_local_GP.put, so we'll go have a look.
*/
initial_put = put;
}
if (send_msgrequest)
xpc_send_chctl_msgrequest_sn2(ch);
}
/*
* Allocate an entry for a message from the message queue associated with the
* specified channel.
*/
static enum xp_retval
xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
struct xpc_msg_sn2 **address_of_msg)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
struct xpc_msg_sn2 *msg;
enum xp_retval ret;
s64 put;
/*
* Get the next available message entry from the local message queue.
* If none are available, we'll make sure that we grab the latest
* GP values.
*/
ret = xpTimeout;
while (1) {
put = ch_sn2->w_local_GP.put;
smp_rmb(); /* guarantee that .put loads before .get */
if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) {
/* There are available message entries. We need to try
* to secure one for ourselves. We'll do this by trying
* to increment w_local_GP.put as long as someone else
* doesn't beat us to it. If they do, we'll have to
* try again.
*/
if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) ==
put) {
/* we got the entry referenced by put */
break;
}
continue; /* try again */
}
/*
* There aren't any available msg entries at this time.
*
* In waiting for a message entry to become available,
* we set a timeout in case the other side is not sending
* completion interrupts. This lets us fake a notify IRQ
* that will cause the notify IRQ handler to fetch the latest
* GP values as if an interrupt was sent by the other side.
*/
if (ret == xpTimeout)
xpc_send_chctl_local_msgrequest_sn2(ch);
if (flags & XPC_NOWAIT)
return xpNoWait;
ret = xpc_allocate_msg_wait(ch);
if (ret != xpInterrupted && ret != xpTimeout)
return ret;
}
/* get the message's address and initialize it */
msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
(put % ch->local_nentries) *
ch->entry_size);
DBUG_ON(msg->flags != 0);
msg->number = put;
dev_dbg(xpc_chan, "w_local_GP.put changed to %lld; msg=0x%p, "
"msg_number=%lld, partid=%d, channel=%d\n", put + 1,
(void *)msg, msg->number, ch->partid, ch->number);
*address_of_msg = msg;
return xpSuccess;
}
/*
* Common code that does the actual sending of the message by advancing the
* local message queue's Put value and sends a chctl msgrequest to the
* partition the message is being sent to.
*/
static enum xp_retval
xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload,
u16 payload_size, u8 notify_type, xpc_notify_func func,
void *key)
{
enum xp_retval ret = xpSuccess;
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
struct xpc_msg_sn2 *msg = msg;
struct xpc_notify_sn2 *notify = notify;
s64 msg_number;
s64 put;
DBUG_ON(notify_type == XPC_N_CALL && func == NULL);
if (XPC_MSG_SIZE(payload_size) > ch->entry_size)
return xpPayloadTooBig;
xpc_msgqueue_ref(ch);
if (ch->flags & XPC_C_DISCONNECTING) {
ret = ch->reason;
goto out_1;
}
if (!(ch->flags & XPC_C_CONNECTED)) {
ret = xpNotConnected;
goto out_1;
}
ret = xpc_allocate_msg_sn2(ch, flags, &msg);
if (ret != xpSuccess)
goto out_1;
msg_number = msg->number;
if (notify_type != 0) {
/*
* Tell the remote side to send an ACK interrupt when the
* message has been delivered.
*/
msg->flags |= XPC_M_SN2_INTERRUPT;
atomic_inc(&ch->n_to_notify);
notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries];
notify->func = func;
notify->key = key;
notify->type = notify_type;
/* ??? Is a mb() needed here? */
if (ch->flags & XPC_C_DISCONNECTING) {
/*
* An error occurred between our last error check and
* this one. We will try to clear the type field from
* the notify entry. If we succeed then
* xpc_disconnect_channel() didn't already process
* the notify entry.
*/
if (cmpxchg(&notify->type, notify_type, 0) ==
notify_type) {
atomic_dec(&ch->n_to_notify);
ret = ch->reason;
}
goto out_1;
}
}
memcpy(&msg->payload, payload, payload_size);
msg->flags |= XPC_M_SN2_READY;
/*
* The preceding store of msg->flags must occur before the following
* load of local_GP->put.
*/
smp_mb();
/* see if the message is next in line to be sent, if so send it */
put = ch_sn2->local_GP->put;
if (put == msg_number)
xpc_send_msgs_sn2(ch, put);
out_1:
xpc_msgqueue_deref(ch);
return ret;
}
/*
* Now we actually acknowledge the messages that have been delivered and ack'd
* by advancing the cached remote message queue's Get value and if requested
* send a chctl msgrequest to the message sender's partition.
*
* If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition
* that sent the message.
*/
static void
xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
{
struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
struct xpc_msg_sn2 *msg;
s64 get = initial_get + 1;
int send_msgrequest = 0;
while (1) {
while (1) {
if (get == ch_sn2->w_local_GP.get)
break;
msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
remote_msgqueue + (get %
ch->remote_nentries) *
ch->entry_size);
if (!(msg->flags & XPC_M_SN2_DONE))
break;
msg_flags |= msg->flags;
get++;
}
if (get == initial_get) {
/* nothing's changed */
break;
}
if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) !=
initial_get) {
/* someone else beat us to it */
DBUG_ON(ch_sn2->local_GP->get <= initial_get);
break;
}
/* we just set the new value of local_GP->get */
dev_dbg(xpc_chan, "local_GP->get changed to %lld, partid=%d, "
"channel=%d\n", get, ch->partid, ch->number);
send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT);
/*
* We need to ensure that the message referenced by
* local_GP->get is not XPC_M_SN2_DONE or that local_GP->get
* equals w_local_GP.get, so we'll go have a look.
*/
initial_get = get;
}
if (send_msgrequest)
xpc_send_chctl_msgrequest_sn2(ch);
}
static void
xpc_received_payload_sn2(struct xpc_channel *ch, void *payload)
{
struct xpc_msg_sn2 *msg;
s64 msg_number;
s64 get;
msg = container_of(payload, struct xpc_msg_sn2, payload);
msg_number = msg->number;
dev_dbg(xpc_chan, "msg=0x%p, msg_number=%lld, partid=%d, channel=%d\n",
(void *)msg, msg_number, ch->partid, ch->number);
DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) !=
msg_number % ch->remote_nentries);
DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
DBUG_ON(msg->flags & XPC_M_SN2_DONE);
msg->flags |= XPC_M_SN2_DONE;
/*
* The preceding store of msg->flags must occur before the following
* load of local_GP->get.
*/
smp_mb();
/*
* See if this message is next in line to be acknowledged as having
* been delivered.
*/
get = ch->sn.sn2.local_GP->get;
if (get == msg_number)
xpc_acknowledge_msgs_sn2(ch, get, msg->flags);
}
static struct xpc_arch_operations xpc_arch_ops_sn2 = {
.setup_partitions = xpc_setup_partitions_sn2,
.teardown_partitions = xpc_teardown_partitions_sn2,
.process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2,
.get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2,
.setup_rsvd_page = xpc_setup_rsvd_page_sn2,
.allow_hb = xpc_allow_hb_sn2,
.disallow_hb = xpc_disallow_hb_sn2,
.disallow_all_hbs = xpc_disallow_all_hbs_sn2,
.increment_heartbeat = xpc_increment_heartbeat_sn2,
.offline_heartbeat = xpc_offline_heartbeat_sn2,
.online_heartbeat = xpc_online_heartbeat_sn2,
.heartbeat_init = xpc_heartbeat_init_sn2,
.heartbeat_exit = xpc_heartbeat_exit_sn2,
.get_remote_heartbeat = xpc_get_remote_heartbeat_sn2,
.request_partition_activation =
xpc_request_partition_activation_sn2,
.request_partition_reactivation =
xpc_request_partition_reactivation_sn2,
.request_partition_deactivation =
xpc_request_partition_deactivation_sn2,
.cancel_partition_deactivation_request =
xpc_cancel_partition_deactivation_request_sn2,
.setup_ch_structures = xpc_setup_ch_structures_sn2,
.teardown_ch_structures = xpc_teardown_ch_structures_sn2,
.make_first_contact = xpc_make_first_contact_sn2,
.get_chctl_all_flags = xpc_get_chctl_all_flags_sn2,
.send_chctl_closerequest = xpc_send_chctl_closerequest_sn2,
.send_chctl_closereply = xpc_send_chctl_closereply_sn2,
.send_chctl_openrequest = xpc_send_chctl_openrequest_sn2,
.send_chctl_openreply = xpc_send_chctl_openreply_sn2,
.send_chctl_opencomplete = xpc_send_chctl_opencomplete_sn2,
.process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2,
.save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2,
.setup_msg_structures = xpc_setup_msg_structures_sn2,
.teardown_msg_structures = xpc_teardown_msg_structures_sn2,
.indicate_partition_engaged = xpc_indicate_partition_engaged_sn2,
.indicate_partition_disengaged = xpc_indicate_partition_disengaged_sn2,
.partition_engaged = xpc_partition_engaged_sn2,
.any_partition_engaged = xpc_any_partition_engaged_sn2,
.assume_partition_disengaged = xpc_assume_partition_disengaged_sn2,
.n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2,
.send_payload = xpc_send_payload_sn2,
.get_deliverable_payload = xpc_get_deliverable_payload_sn2,
.received_payload = xpc_received_payload_sn2,
.notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2,
};
int
xpc_init_sn2(void)
{
int ret;
size_t buf_size;
xpc_arch_ops = xpc_arch_ops_sn2;
if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) {
dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is "
"larger than %d\n", XPC_MSG_HDR_MAX_SIZE);
return -E2BIG;
}
buf_size = max(XPC_RP_VARS_SIZE,
XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2);
xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size,
GFP_KERNEL,
&xpc_remote_copy_buffer_base_sn2);
if (xpc_remote_copy_buffer_sn2 == NULL) {
dev_err(xpc_part, "can't get memory for remote copy buffer\n");
return -ENOMEM;
}
/* open up protections for IPI and [potentially] amo operations */
xpc_allow_IPI_ops_sn2();
xpc_allow_amo_ops_shub_wars_1_1_sn2();
/*
* This is safe to do before the xpc_hb_checker thread has started
* because the handler releases a wait queue. If an interrupt is
* received before the thread is waiting, it will not go to sleep,
* but rather immediately process the interrupt.
*/
ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0,
"xpc hb", NULL);
if (ret != 0) {
dev_err(xpc_part, "can't register ACTIVATE IRQ handler, "
"errno=%d\n", -ret);
xpc_disallow_IPI_ops_sn2();
kfree(xpc_remote_copy_buffer_base_sn2);
}
return ret;
}
void
xpc_exit_sn2(void)
{
free_irq(SGI_XPC_ACTIVATE, NULL);
xpc_disallow_IPI_ops_sn2();
kfree(xpc_remote_copy_buffer_base_sn2);
}