psci_common.c

/*
 * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * Neither the name of ARM nor the names of its contributors may be used
 * to endorse or promote products derived from this software without specific
 * prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <arch.h>
#include <arch_helpers.h>
#include <assert.h>
#include <bl_common.h>
#include <context.h>
#include <context_mgmt.h>
#include <debug.h>
#include <platform.h>
#include <string.h>
#include "psci_private.h"

/*
 * SPD power management operations, expected to be supplied by the registered
 * SPD on successful SP initialization
 */
const spd_pm_ops_t *psci_spd_pm;

/*******************************************************************************
 * Grand array that holds the platform's topology information for state
 * management of affinity instances. Each node (aff_map_node) in the array
 * corresponds to an affinity instance e.g. cluster, cpu within an mpidr
 ******************************************************************************/
aff_map_node_t psci_aff_map[PSCI_NUM_AFFS]
#if USE_COHERENT_MEM
__attribute__ ((section("tzfw_coherent_mem")))
#endif
;

/*******************************************************************************
 * Pointer to functions exported by the platform to complete power mgmt. ops
 ******************************************************************************/
const plat_pm_ops_t *psci_plat_pm_ops;

/*******************************************************************************
 * This function is passed an array of pointers to affinity level nodes in the
 * topology tree for an mpidr. It iterates through the nodes to find the highest
 * affinity level which is marked as physically powered off.
 ******************************************************************************/
uint32_t psci_find_max_phys_off_afflvl(uint32_t start_afflvl,
				       uint32_t end_afflvl,
				       aff_map_node_t *mpidr_nodes[])
{
	uint32_t max_afflvl = PSCI_INVALID_DATA;

	for (; start_afflvl <= end_afflvl; start_afflvl++) {
		if (mpidr_nodes[start_afflvl] == NULL)
			continue;

		if (psci_get_phys_state(mpidr_nodes[start_afflvl]) ==
		    PSCI_STATE_OFF)
			max_afflvl = start_afflvl;
	}

	return max_afflvl;
}

/*******************************************************************************
 * This function saves the highest affinity level which is in OFF state. The
 * affinity instance with which the level is associated is determined by the
 * caller.
 ******************************************************************************/
void psci_set_max_phys_off_afflvl(uint32_t afflvl)
{
	set_cpu_data(psci_svc_cpu_data.max_phys_off_afflvl, afflvl);

	/*
	 * Ensure that the saved value is flushed to main memory and any
	 * speculatively pre-fetched stale copies are invalidated from the
	 * caches of other cpus in the same coherency domain. This ensures that
	 * the value can be safely read irrespective of the state of the data
	 * cache.
	 */
	flush_cpu_data(psci_svc_cpu_data.max_phys_off_afflvl);
}

/*******************************************************************************
 * This function reads the saved highest affinity level which is in OFF
 * state. The affinity instance with which the level is associated is determined
 * by the caller.
 ******************************************************************************/
uint32_t psci_get_max_phys_off_afflvl(void)
{
	/*
	 * Ensure that the last update of this value in this cpu's cache is
	 * flushed to main memory and any speculatively pre-fetched stale copies
	 * are invalidated from the caches of other cpus in the same coherency
	 * domain. This ensures that the value is always read from the main
	 * memory when it was written before the data cache was enabled.
	 */
	flush_cpu_data(psci_svc_cpu_data.max_phys_off_afflvl);
	return get_cpu_data(psci_svc_cpu_data.max_phys_off_afflvl);
}

/*******************************************************************************
 * Routine to return the maximum affinity level to traverse to after a cpu has
 * been physically powered up. It is expected to be called immediately after
 * reset from assembler code.
 ******************************************************************************/
int get_power_on_target_afflvl()
{
	int afflvl;

#if DEBUG
	unsigned int state;
	aff_map_node_t *node;

	/* Retrieve our node from the topology tree */
	node = psci_get_aff_map_node(read_mpidr_el1() & MPIDR_AFFINITY_MASK,
				     MPIDR_AFFLVL0);
	assert(node);

	/*
	 * Sanity check the state of the cpu. It should be either suspend or "on
	 * pending"
	 */
	state = psci_get_state(node);
	assert(state == PSCI_STATE_SUSPEND || state == PSCI_STATE_ON_PENDING);
#endif

	/*
	 * Assume that this cpu was suspended and retrieve its target affinity
	 * level. If it is invalid then it could only have been turned off
	 * earlier. get_max_afflvl() will return the highest affinity level a
	 * cpu can be turned off to.
	 */
	afflvl = psci_get_suspend_afflvl();
	if (afflvl == PSCI_INVALID_DATA)
		afflvl = get_max_afflvl();
	return afflvl;
}

/*******************************************************************************
 * Simple routine to retrieve the maximum affinity level supported by the
 * platform and check that it makes sense.
 ******************************************************************************/
int get_max_afflvl(void)
{
	int aff_lvl;

	aff_lvl = plat_get_max_afflvl();
	assert(aff_lvl <= MPIDR_MAX_AFFLVL && aff_lvl >= MPIDR_AFFLVL0);

	return aff_lvl;
}

/*******************************************************************************
 * Simple routine to set the id of an affinity instance at a given level in the
 * mpidr.
 ******************************************************************************/
unsigned long mpidr_set_aff_inst(unsigned long mpidr,
				 unsigned char aff_inst,
				 int aff_lvl)
{
	unsigned long aff_shift;

	assert(aff_lvl <= MPIDR_AFFLVL3);

	/*
	 * Decide the number of bits to shift by depending upon
	 * the affinity level
	 */
	aff_shift = get_afflvl_shift(aff_lvl);

	/* Clear the existing affinity instance & set the new one*/
	mpidr &= ~(MPIDR_AFFLVL_MASK << aff_shift);
	mpidr |= aff_inst << aff_shift;

	return mpidr;
}

/*******************************************************************************
 * This function sanity checks a range of affinity levels.
 ******************************************************************************/
int psci_check_afflvl_range(int start_afflvl, int end_afflvl)
{
	/* Sanity check the parameters passed */
	if (end_afflvl > get_max_afflvl())
		return PSCI_E_INVALID_PARAMS;

	if (start_afflvl < MPIDR_AFFLVL0)
		return PSCI_E_INVALID_PARAMS;

	if (end_afflvl < start_afflvl)
		return PSCI_E_INVALID_PARAMS;

	return PSCI_E_SUCCESS;
}

/*******************************************************************************
 * This function is passed an array of pointers to affinity level nodes in the
 * topology tree for an mpidr and the state which each node should transition
 * to. It updates the state of each node between the specified affinity levels.
 ******************************************************************************/
void psci_do_afflvl_state_mgmt(uint32_t start_afflvl,
			       uint32_t end_afflvl,
			       aff_map_node_t *mpidr_nodes[],
			       uint32_t state)
{
	uint32_t level;

	for (level = start_afflvl; level <= end_afflvl; level++) {
		if (mpidr_nodes[level] == NULL)
			continue;
		psci_set_state(mpidr_nodes[level], state);
	}
}

/*******************************************************************************
 * This function is passed an array of pointers to affinity level nodes in the
 * topology tree for an mpidr. It picks up locks for each affinity level bottom
 * up in the range specified.
 ******************************************************************************/
void psci_acquire_afflvl_locks(int start_afflvl,
			       int end_afflvl,
			       aff_map_node_t *mpidr_nodes[])
{
	int level;

	for (level = start_afflvl; level <= end_afflvl; level++) {
		if (mpidr_nodes[level] == NULL)
			continue;

		psci_lock_get(mpidr_nodes[level]);
	}
}

/*******************************************************************************
 * This function is passed an array of pointers to affinity level nodes in the
 * topology tree for an mpidr. It releases the lock for each affinity level top
 * down in the range specified.
 ******************************************************************************/
void psci_release_afflvl_locks(int start_afflvl,
			       int end_afflvl,
			       aff_map_node_t *mpidr_nodes[])
{
	int level;

	for (level = end_afflvl; level >= start_afflvl; level--) {
		if (mpidr_nodes[level] == NULL)
			continue;

		psci_lock_release(mpidr_nodes[level]);
	}
}

/*******************************************************************************
 * Simple routine to determine whether an affinity instance at a given level
 * in an mpidr exists or not.
 ******************************************************************************/
int psci_validate_mpidr(unsigned long mpidr, int level)
{
	aff_map_node_t *node;

	node = psci_get_aff_map_node(mpidr, level);
	if (node && (node->state & PSCI_AFF_PRESENT))
		return PSCI_E_SUCCESS;
	else
		return PSCI_E_INVALID_PARAMS;
}

/*******************************************************************************
 * This function determines the full entrypoint information for the requested
 * PSCI entrypoint on power on/resume and saves this in the non-secure CPU
 * cpu_context, ready for when the core boots.
 ******************************************************************************/
int psci_save_ns_entry(uint64_t mpidr,
		       uint64_t entrypoint, uint64_t context_id,
		       uint32_t ns_scr_el3, uint32_t ns_sctlr_el1)
{
	uint32_t ep_attr, mode, sctlr, daif, ee;
	entry_point_info_t ep;

	sctlr = ns_scr_el3 & SCR_HCE_BIT ? read_sctlr_el2() : ns_sctlr_el1;
	ee = 0;

	ep_attr = NON_SECURE | EP_ST_DISABLE;
	if (sctlr & SCTLR_EE_BIT) {
		ep_attr |= EP_EE_BIG;
		ee = 1;
	}
	SET_PARAM_HEAD(&ep, PARAM_EP, VERSION_1, ep_attr);

	ep.pc = entrypoint;
	memset(&ep.args, 0, sizeof(ep.args));
	ep.args.arg0 = context_id;

	/*
	 * Figure out whether the cpu enters the non-secure address space
	 * in aarch32 or aarch64
	 */
	if (ns_scr_el3 & SCR_RW_BIT) {

		/*
		 * Check whether a Thumb entry point has been provided for an
		 * aarch64 EL
		 */
		if (entrypoint & 0x1)
			return PSCI_E_INVALID_PARAMS;

		mode = ns_scr_el3 & SCR_HCE_BIT ? MODE_EL2 : MODE_EL1;

		ep.spsr = SPSR_64(mode, MODE_SP_ELX, DISABLE_ALL_EXCEPTIONS);
	} else {

		mode = ns_scr_el3 & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc;

		/*
		 * TODO: Choose async. exception bits if HYP mode is not
		 * implemented according to the values of SCR.{AW, FW} bits
		 */
		daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT;

		ep.spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, daif);
	}

	/* initialise an entrypoint to set up the CPU context */
	cm_init_context(mpidr, &ep);

	return PSCI_E_SUCCESS;
}

/*******************************************************************************
 * This function takes a pointer to an affinity node in the topology tree and
 * returns its state. State of a non-leaf node needs to be calculated.
 ******************************************************************************/
unsigned short psci_get_state(aff_map_node_t *node)
{
#if !USE_COHERENT_MEM
	flush_dcache_range((uint64_t) node, sizeof(*node));
#endif

	assert(node->level >= MPIDR_AFFLVL0 && node->level <= MPIDR_MAX_AFFLVL);

	/* A cpu node just contains the state which can be directly returned */
	if (node->level == MPIDR_AFFLVL0)
		return (node->state >> PSCI_STATE_SHIFT) & PSCI_STATE_MASK;

	/*
	 * For an affinity level higher than a cpu, the state has to be
	 * calculated. It depends upon the value of the reference count
	 * which is managed by each node at the next lower affinity level
	 * e.g. for a cluster, each cpu increments/decrements the reference
	 * count. If the reference count is 0 then the affinity level is
	 * OFF else ON.
	 */
	if (node->ref_count)
		return PSCI_STATE_ON;
	else
		return PSCI_STATE_OFF;
}

/*******************************************************************************
 * This function takes a pointer to an affinity node in the topology tree and
 * a target state. State of a non-leaf node needs to be converted to a reference
 * count. State of a leaf node can be set directly.
 ******************************************************************************/
void psci_set_state(aff_map_node_t *node, unsigned short state)
{
	assert(node->level >= MPIDR_AFFLVL0 && node->level <= MPIDR_MAX_AFFLVL);

	/*
	 * For an affinity level higher than a cpu, the state is used
	 * to decide whether the reference count is incremented or
	 * decremented. Entry into the ON_PENDING state does not have
	 * effect.
	 */
	if (node->level > MPIDR_AFFLVL0) {
		switch (state) {
		case PSCI_STATE_ON:
			node->ref_count++;
			break;
		case PSCI_STATE_OFF:
		case PSCI_STATE_SUSPEND:
			node->ref_count--;
			break;
		case PSCI_STATE_ON_PENDING:
			/*
			 * An affinity level higher than a cpu will not undergo
			 * a state change when it is about to be turned on
			 */
			return;
		default:
			assert(0);
		}
	} else {
		node->state &= ~(PSCI_STATE_MASK << PSCI_STATE_SHIFT);
		node->state |= (state & PSCI_STATE_MASK) << PSCI_STATE_SHIFT;
	}

#if !USE_COHERENT_MEM
	flush_dcache_range((uint64_t) node, sizeof(*node));
#endif
}

/*******************************************************************************
 * An affinity level could be on, on_pending, suspended or off. These are the
 * logical states it can be in. Physically either it is off or on. When it is in
 * the state on_pending then it is about to be turned on. It is not possible to
 * tell whether that's actually happenned or not. So we err on the side of
 * caution & treat the affinity level as being turned off.
 ******************************************************************************/
unsigned short psci_get_phys_state(aff_map_node_t *node)
{
	unsigned int state;

	state = psci_get_state(node);
	return get_phys_state(state);
}

/*******************************************************************************
 * This function takes an array of pointers to affinity instance nodes in the
 * topology tree and calls the physical power on handler for the corresponding
 * affinity levels
 ******************************************************************************/
static int psci_call_power_on_handlers(aff_map_node_t *mpidr_nodes[],
				       int start_afflvl,
				       int end_afflvl,
				       afflvl_power_on_finisher_t *pon_handlers)
{
	int rc = PSCI_E_INVALID_PARAMS, level;
	aff_map_node_t *node;

	for (level = end_afflvl; level >= start_afflvl; level--) {
		node = mpidr_nodes[level];
		if (node == NULL)
			continue;

		/*
		 * If we run into any trouble while powering up an
		 * affinity instance, then there is no recovery path
		 * so simply return an error and let the caller take
		 * care of the situation.
		 */
		rc = pon_handlers[level](node);
		if (rc != PSCI_E_SUCCESS)
			break;
	}

	return rc;
}

/*******************************************************************************
 * Generic handler which is called when a cpu is physically powered on. It
 * traverses through all the affinity levels performing generic, architectural,
 * platform setup and state management e.g. for a cluster that's been powered
 * on, it will call the platform specific code which will enable coherency at
 * the interconnect level. For a cpu it could mean turning on the MMU etc.
 *
 * The state of all the relevant affinity levels is changed after calling the
 * affinity level specific handlers as their actions would depend upon the state
 * the affinity level is exiting from.
 *
 * The affinity level specific handlers are called in descending order i.e. from
 * the highest to the lowest affinity level implemented by the platform because
 * to turn on affinity level X it is neccesary to turn on affinity level X + 1
 * first.
 ******************************************************************************/
void psci_afflvl_power_on_finish(int start_afflvl,
				 int end_afflvl,
				 afflvl_power_on_finisher_t *pon_handlers)
{
	mpidr_aff_map_nodes_t mpidr_nodes;
	int rc;
	unsigned int max_phys_off_afflvl;


	/*
	 * Collect the pointers to the nodes in the topology tree for
	 * each affinity instance in the mpidr. If this function does
	 * not return successfully then either the mpidr or the affinity
	 * levels are incorrect. Either case is an irrecoverable error.
	 */
	rc = psci_get_aff_map_nodes(read_mpidr_el1() & MPIDR_AFFINITY_MASK,
				    start_afflvl,
				    end_afflvl,
				    mpidr_nodes);
	if (rc != PSCI_E_SUCCESS)
		panic();

	/*
	 * This function acquires the lock corresponding to each affinity
	 * level so that by the time all locks are taken, the system topology
	 * is snapshot and state management can be done safely.
	 */
	psci_acquire_afflvl_locks(start_afflvl,
				  end_afflvl,
				  mpidr_nodes);

	max_phys_off_afflvl = psci_find_max_phys_off_afflvl(start_afflvl,
							    end_afflvl,
							    mpidr_nodes);
	assert(max_phys_off_afflvl != PSCI_INVALID_DATA);

	/*
	 * Stash the highest affinity level that will come out of the OFF or
	 * SUSPEND states.
	 */
	psci_set_max_phys_off_afflvl(max_phys_off_afflvl);

	/* Perform generic, architecture and platform specific handling */
	rc = psci_call_power_on_handlers(mpidr_nodes,
					 start_afflvl,
					 end_afflvl,
					 pon_handlers);
	if (rc != PSCI_E_SUCCESS)
		panic();

	/*
	 * This function updates the state of each affinity instance
	 * corresponding to the mpidr in the range of affinity levels
	 * specified.
	 */
	psci_do_afflvl_state_mgmt(start_afflvl,
				  end_afflvl,
				  mpidr_nodes,
				  PSCI_STATE_ON);

	/*
	 * Invalidate the entry for the highest affinity level stashed earlier.
	 * This ensures that any reads of this variable outside the power
	 * up/down sequences return PSCI_INVALID_DATA
	 */
	psci_set_max_phys_off_afflvl(PSCI_INVALID_DATA);

	/*
	 * This loop releases the lock corresponding to each affinity level
	 * in the reverse order to which they were acquired.
	 */
	psci_release_afflvl_locks(start_afflvl,
				  end_afflvl,
				  mpidr_nodes);
}

/*******************************************************************************
 * This function initializes the set of hooks that PSCI invokes as part of power
 * management operation. The power management hooks are expected to be provided
 * by the SPD, after it finishes all its initialization
 ******************************************************************************/
void psci_register_spd_pm_hook(const spd_pm_ops_t *pm)
{
	psci_spd_pm = pm;
}

/*******************************************************************************
 * This function prints the state of all affinity instances present in the
 * system
 ******************************************************************************/
void psci_print_affinity_map(void)
{
#if LOG_LEVEL >= LOG_LEVEL_INFO
	aff_map_node_t *node;
	unsigned int idx;
	/* This array maps to the PSCI_STATE_X definitions in psci.h */
	static const char *psci_state_str[] = {
		"ON",
		"OFF",
		"ON_PENDING",
		"SUSPEND"
	};

	INFO("PSCI Affinity Map:\n");
	for (idx = 0; idx < PSCI_NUM_AFFS ; idx++) {
		node = &psci_aff_map[idx];
		if (!(node->state & PSCI_AFF_PRESENT)) {
			continue;
		}
		INFO("  AffInst: Level %u, MPID 0x%lx, State %s\n",
				node->level, node->mpidr,
				psci_state_str[psci_get_state(node)]);
	}
#endif
}