Commit 8b6e511e authored by Cliff Wickman's avatar Cliff Wickman Committed by Ingo Molnar
Browse files

x86/uv: Work around UV2 BAU hangs 



On SGI's UV2 the BAU (Broadcast Assist Unit) driver can hang
under a heavy load. To cure this:

- Disable the UV2 extended status mode (see UV2_EXT_SHFT), as
  this mode changes BAU behavior in more ways then just delivering
  an extra bit of status.  Revert status to just two meaningful bits,
  like UV1.

- Use no IPI-style resets on UV2.  Just give up the request for
  whatever the reason it failed and let it be accomplished with
  the legacy IPI method.

- Use no alternate sending descriptor (the former UV2 workaround
  bcp->using_desc and handle_uv2_busy() stuff).  Just disable the
  use of the BAU for a period of time in favor of the legacy IPI
  method when the h/w bug leaves a descriptor busy.

  -- new tunable: giveup_limit determines the threshold at which a hub is
     so plugged that it should do all requests with the legacy IPI method for a
     period of time
  -- generalize disable_for_congestion() (renamed disable_for_period()) for
     use whenever a hub should avoid using the BAU for a period of time

Also:

 - Fix find_another_by_swack(), which is part of the UV2 bug workaround

 - Correct and clarify the statistics (new stats s_overipilimit, s_giveuplimit,
   s_enters, s_ipifordisabled, s_plugged, s_congested)

Signed-off-by: default avatarCliff Wickman <cpw@sgi.com>
Link: http://lkml.kernel.org/r/20120622131459.GC31884@sgi.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 26ef8577

arch/x86/include/asm/uv/uv_bau.h

+18 −10
Original line number Diff line number Diff line
@@ -140,6 +140,9 @@ 
#define IPI_RESET_LIMIT			1

/* after this # consecutive successes, bump up the throttle if it was lowered */

#define COMPLETE_THRESHOLD		5

/* after this # of giveups (fall back to kernel IPI's) disable the use of

   the BAU for a period of time */

#define GIVEUP_LIMIT			100



#define UV_LB_SUBNODEID			0x10
@@ -166,7 +169,6 @@ 
#define FLUSH_RETRY_TIMEOUT		2

#define FLUSH_GIVEUP			3

#define FLUSH_COMPLETE			4

#define FLUSH_RETRY_BUSYBUG		5



/*

 * tuning the action when the numalink network is extremely delayed
@@ -175,7 +177,7 @@ 
						   microseconds */

#define CONGESTED_REPS			10	/* long delays averaged over

						   this many broadcasts */

#define CONGESTED_PERIOD		30	/* time for the bau to be

#define DISABLED_PERIOD			10	/* time for the bau to be

						   disabled, in seconds */

/* see msg_type: */

#define MSG_NOOP			0
@@ -520,7 +522,12 @@ struct ptc_stats { 
	unsigned long	s_uv2_wars;		/* uv2 workaround, perm. busy */

	unsigned long	s_uv2_wars_hw;		/* uv2 workaround, hiwater */

	unsigned long	s_uv2_war_waits;	/* uv2 workaround, long waits */

	unsigned long	s_overipilimit;		/* over the ipi reset limit */

	unsigned long	s_giveuplimit;		/* disables, over giveup limit*/

	unsigned long	s_enters;		/* entries to the driver */

	unsigned long	s_ipifordisabled;	/* fall back to IPI; disabled */

	unsigned long	s_plugged;		/* plugged by h/w bug*/

	unsigned long	s_congested;		/* giveup on long wait */

	/* destination statistics */

	unsigned long	d_alltlb;		/* times all tlb's on this

						   cpu were flushed */
@@ -588,8 +595,7 @@ struct bau_control { 
	int			ipi_attempts;

	int			conseccompletes;

	short			nobau;

	int			baudisabled;

	int			set_bau_off;

	short			baudisabled;

	short			cpu;

	short			osnode;

	short			uvhub_cpu;
@@ -598,14 +604,16 @@ struct bau_control { 
	short			cpus_in_socket;

	short			cpus_in_uvhub;

	short			partition_base_pnode;

	short			using_desc; /* an index, like uvhub_cpu */

	unsigned int		inuse_map;

	short			busy;       /* all were busy (war) */

	unsigned short		message_number;

	unsigned short		uvhub_quiesce;

	short			socket_acknowledge_count[DEST_Q_SIZE];

	cycles_t		send_message;

	cycles_t		period_end;

	cycles_t		period_time;

	spinlock_t		uvhub_lock;

	spinlock_t		queue_lock;

	spinlock_t		disable_lock;

	/* tunables */

	int			max_concurr;

	int			max_concurr_const;
@@ -616,9 +624,9 @@ struct bau_control { 
	int			complete_threshold;

	int			cong_response_us;

	int			cong_reps;

	int			cong_period;

	unsigned long		clocks_per_100_usec;

	cycles_t		period_time;

	cycles_t		disabled_period;

	int			period_giveups;

	int			giveup_limit;

	long			period_requests;

	struct hub_and_pnode	*thp;

};

arch/x86/platform/uv/tlb_uv.c

+182 −205

File changed.

Preview size limit exceeded, changes collapsed.

CRA Git | Maintained and supported by SUSTech CRA and CCSE