esp-idf/components/esp_hw_support/mspi_timing_tuning.c

/*
 * SPDX-FileCopyrightText: 2019-2023 Espressif Systems (Shanghai) CO LTD
 *
 * SPDX-License-Identifier: Apache-2.0
 */

#include <stdlib.h>
#include <string.h>
#include "sdkconfig.h"
#include "esp_attr.h"
#include "esp_err.h"
#include "esp_types.h"
#include "esp_log.h"
#include "soc/io_mux_reg.h"
#include "soc/soc.h"
#include "hal/spi_flash_hal.h"
#include "hal/cache_hal.h"
#include "esp_private/mspi_timing_tuning.h"
#include "mspi_timing_config.h"
#include "mspi_timing_by_mspi_delay.h"
#if SOC_MEMSPI_TIMING_TUNING_BY_MSPI_DELAY
#include "mspi_timing_tuning_configs.h"
#include "hal/mspi_timing_tuning_ll.h"
#endif
#if SOC_MEMSPI_CLK_SRC_IS_INDEPENDENT
#include "hal/spimem_flash_ll.h"
#endif

#if CONFIG_ESPTOOLPY_FLASHFREQ_120M
#define FLASH_FREQUENCY_MHZ 120
#elif CONFIG_ESPTOOLPY_FLASHFREQ_80M
#define FLASH_FREQUENCY_MHZ 80
#elif CONFIG_ESPTOOLPY_FLASHFREQ_64M
#define FLASH_FREQUENCY_MHZ 64
#elif CONFIG_ESPTOOLPY_FLASHFREQ_60M
#define FLASH_FREQUENCY_MHZ 60
#elif CONFIG_ESPTOOLPY_FLASHFREQ_48M
#define FLASH_FREQUENCY_MHZ 48
#elif CONFIG_ESPTOOLPY_FLASHFREQ_40M
#define FLASH_FREQUENCY_MHZ 40
#elif CONFIG_ESPTOOLPY_FLASHFREQ_32M
#define FLASH_FREQUENCY_MHZ 32
#elif CONFIG_ESPTOOLPY_FLASHFREQ_30M
#define FLASH_FREQUENCY_MHZ 30
#elif CONFIG_ESPTOOLPY_FLASHFREQ_26M
#define FLASH_FREQUENCY_MHZ 26
#elif CONFIG_ESPTOOLPY_FLASHFREQ_24M
#define FLASH_FREQUENCY_MHZ 24
#elif CONFIG_ESPTOOLPY_FLASHFREQ_20M
#define FLASH_FREQUENCY_MHZ 20
#elif CONFIG_ESPTOOLPY_FLASHFREQ_16M
#define FLASH_FREQUENCY_MHZ 16
#elif CONFIG_ESPTOOLPY_FLASHFREQ_15M
#define FLASH_FREQUENCY_MHZ 15
#endif

/**
 * @brief MSPI timing tuning type
 */
typedef enum {
    MSPI_TIMING_TUNING_MSPI_DIN_DUMMY,    //tune by mspi din and dummy
} mspi_timing_tuning_t;

typedef struct mspi_tuning_cfg_drv_s mspi_tuning_cfg_drv_t;
__attribute__((unused)) const static char *TAG = "MSPI Timing";

struct mspi_tuning_cfg_drv_s {

    /**
     * @brief Flash tuning scheme type
     */
    mspi_timing_tuning_t flash_tuning_type;

    /**
     * @brief Init MSPI for Flash timing tuning
     *
     * @param[in] flash_freq_mhz  Flash frequency in MHz
     */
    void (*flash_init_mspi)(uint32_t flash_freq_mhz);

    /**
     * @brief Configure MSPI for Flash timing tuning
     *
     * @param[in] params  Timing tuning parameters
     */
    void (*flash_tune_mspi)(const void *params);

    /**
     * @brief Flash read
     *
     * @param[in] buf   Read buffer
     * @param[in] addr  Read address
     * @param[in] len   Read length
     */
    void (*flash_read)(uint8_t *buf, uint32_t addr, uint32_t len);

    /**
     * @brief Select best tuning configs for Flash
     *
     * @param[in] configs             Timing tuning configurations
     * @param[in] consecutive_length  Length of the consecutive successful sample results
     * @param[in] end                 End of the consecutive successful sample results
     * @param[in] reference_data      Reference data
     * @param[in] is_ddr              DDR or SDR
     *
     * @return  Best config ID
     */
    uint32_t (*flash_select_best_tuning_config)(const void *configs, uint32_t consecutive_length, uint32_t end, const uint8_t *reference_data, bool is_ddr);

    /**
     * @brief Set best Flash tuning configs.
     *        After this, calling `mspi_timing_enter_high_speed_mode` will set these configs correctly
     *
     * @param[in] params  Timing tuning parameters
     */
    void (*flash_set_best_tuning_config)(const void *params);

    /**
     * @brief PSRAM tuning scheme type
     */
    mspi_timing_tuning_t psram_tuning_type;

    /**
     * @brief Init MSPI for PSRAM timing tuning
     *
     * @param[in] flash_freq_mhz  PSRAM frequency in MHz
     */
    void (*psram_init_mspi)(uint32_t psram_freq_mhz);

    /**
     * @brief Configure MSPI for PSRAM timing tuning
     *
     * @param[in] params  Timing tuning parameters
     */
    void (*psram_tune_mspi)(const void *params);

    /**
     * @brief PSRAM read
     *
     * @param[in] buf   Read buffer
     * @param[in] addr  Read address
     * @param[in] len   Read length
     */
    void (*psram_read)(uint8_t *buf, uint32_t addr, uint32_t len);

    /**
     * @brief Select best tuning configs for PSRAM
     *
     * @param[in] configs             Timing tuning configurations
     * @param[in] consecutive_length  Length of the consecutive successful sample results
     * @param[in] end                 End of the consecutive successful sample results
     * @param[in] reference_data      Reference data
     * @param[in] is_ddr              DDR or SDR
     *
     * @return  Best config ID
     */
    uint32_t (*psram_select_best_tuning_config)(const void *configs, uint32_t consecutive_length, uint32_t end, const uint8_t *reference_data, bool is_ddr);

    /**
     * @brief Set best PSRAM tuning configs.
     *        After this, calling `mspi_timing_enter_high_speed_mode` will set these configs correctly
     *
     * @param[in] params  Timing tuning parameters
     */
    void (*psram_set_best_tuning_config)(const void *params);
};

static mspi_tuning_cfg_drv_t s_tuning_cfg_drv = {};

void s_register_config_driver(mspi_tuning_cfg_drv_t *cfg_drv, bool is_flash)
{
    if (is_flash) {
        s_tuning_cfg_drv.flash_tuning_type = cfg_drv->flash_tuning_type;
        s_tuning_cfg_drv.flash_init_mspi = cfg_drv->flash_init_mspi;
        s_tuning_cfg_drv.flash_tune_mspi = cfg_drv->flash_tune_mspi;
        s_tuning_cfg_drv.flash_read = cfg_drv->flash_read;
        s_tuning_cfg_drv.flash_select_best_tuning_config = cfg_drv->flash_select_best_tuning_config;
        s_tuning_cfg_drv.flash_set_best_tuning_config = cfg_drv->flash_set_best_tuning_config;
    } else {
        s_tuning_cfg_drv.psram_tuning_type = cfg_drv->psram_tuning_type;
        s_tuning_cfg_drv.psram_init_mspi = cfg_drv->psram_init_mspi;
        s_tuning_cfg_drv.psram_tune_mspi = cfg_drv->psram_tune_mspi;
        s_tuning_cfg_drv.psram_read = cfg_drv->psram_read;
        s_tuning_cfg_drv.psram_select_best_tuning_config = cfg_drv->psram_select_best_tuning_config;
        s_tuning_cfg_drv.psram_set_best_tuning_config = cfg_drv->psram_set_best_tuning_config;
    }
}

#if MSPI_TIMING_FLASH_NEEDS_TUNING || MSPI_TIMING_PSRAM_NEEDS_TUNING
/**
 * We use different MSPI timing tuning config to read data to see if current MSPI sampling is successful.
 * The sampling result will be stored in an array. In this array, successful item will be 1, failed item will be 0.
 */
static void s_sweep_for_success_sample_points(uint8_t *reference_data, void *config, bool is_flash, uint8_t *out_array)
{
    const mspi_timing_config_t *timing_config = (const mspi_timing_config_t *)config;
    uint32_t config_idx = 0;
    uint8_t read_data[MSPI_TIMING_TEST_DATA_LEN] = {0};

    for (config_idx = 0; config_idx < timing_config->available_config_num; config_idx++) {
        memset(read_data, 0, MSPI_TIMING_TEST_DATA_LEN);
#if MSPI_TIMING_FLASH_NEEDS_TUNING
        if (is_flash) {
            s_tuning_cfg_drv.flash_tune_mspi(&(timing_config->tuning_config_table[config_idx]));
            s_tuning_cfg_drv.flash_read(read_data, MSPI_TIMING_FLASH_TEST_DATA_ADDR, sizeof(read_data));
        }
#endif
#if MSPI_TIMING_PSRAM_NEEDS_TUNING
        if (!is_flash) {
            s_tuning_cfg_drv.psram_tune_mspi(&(timing_config->tuning_config_table[config_idx]));
            s_tuning_cfg_drv.psram_read(read_data, MSPI_TIMING_PSRAM_TEST_DATA_ADDR, MSPI_TIMING_TEST_DATA_LEN);
        }
#endif
        if (memcmp(reference_data, read_data, sizeof(read_data)) == 0) {
            out_array[config_idx] = 1;
            ESP_EARLY_LOGD(TAG, "%d, good", config_idx);
        } else {
            ESP_EARLY_LOGD(TAG, "%d, bad", config_idx);
        }

    }
}

/**
 * Find consecutive successful sampling points.
 * e.g. array: {1, 1, 0, 0, 1, 1, 1, 0}
 * out_length: 3
 * outout_end_index: 6
 */
static void s_find_max_consecutive_success_points(uint8_t *array, uint32_t size, uint32_t *out_length, uint32_t *out_end_index)
{
    uint32_t max = 0;
    uint32_t match_num = 0;
    uint32_t i = 0;
    uint32_t end = 0;

    while (i < size) {
        if (array[i]) {
            match_num++;
        } else {
            if (match_num > max) {
                max = match_num;
                end = i - 1;
            }
            match_num = 0;
        }
        i++;
    }

    *out_length = match_num > max ? match_num : max;
    *out_end_index = match_num == size ? size : end;
}

static void s_select_best_tuning_config(mspi_timing_config_t *config, uint32_t consecutive_length, uint32_t end, const uint8_t *reference_data, bool is_flash)
{
    const mspi_timing_config_t *timing_config = (const mspi_timing_config_t *)config;
    uint32_t best_point = 0;
    if (is_flash) {
#if MSPI_TIMING_FLASH_DTR_MODE
        best_point = s_tuning_cfg_drv.flash_select_best_tuning_config(timing_config, consecutive_length, end, reference_data, IS_DDR);
#elif MSPI_TIMING_FLASH_STR_MODE
        best_point = s_tuning_cfg_drv.flash_select_best_tuning_config(timing_config, consecutive_length, end, NULL, IS_SDR);
#endif
        s_tuning_cfg_drv.flash_set_best_tuning_config(&(timing_config->tuning_config_table[best_point]));
    } else {
#if MSPI_TIMING_PSRAM_DTR_MODE
        best_point = s_tuning_cfg_drv.psram_select_best_tuning_config(timing_config, consecutive_length, end, reference_data, IS_DDR);
#elif MSPI_TIMING_PSRAM_STR_MODE
        best_point = s_tuning_cfg_drv.psram_select_best_tuning_config(timing_config, consecutive_length, end, NULL, IS_SDR);
#endif
        s_tuning_cfg_drv.psram_set_best_tuning_config(&(timing_config->tuning_config_table[best_point]));
    }
}

static void s_do_tuning(uint8_t *reference_data, void *timing_config, bool is_flash)
{
    /**
     * We use MSPI to tune the timing:
     * 1. Get all MSPI sampling results.
     * 2. Find the longest consecutive successful sampling points from the result above.
     * 3. The middle one will be the best sampling point.
     */
    uint32_t consecutive_length = 0;
    uint32_t last_success_point = 0;
    uint8_t sample_result[MSPI_TIMING_CONFIG_NUM_DEFAULT] = {0};

#if MSPI_TIMING_FLASH_NEEDS_TUNING
    if (is_flash) {
        s_tuning_cfg_drv.flash_init_mspi(FLASH_FREQUENCY_MHZ);
    }
#endif
#if MSPI_TIMING_PSRAM_NEEDS_TUNING
    if (!is_flash) {
        s_tuning_cfg_drv.psram_init_mspi(CONFIG_SPIRAM_SPEED);
    }
#endif

    s_sweep_for_success_sample_points(reference_data, timing_config, is_flash, sample_result);
    s_find_max_consecutive_success_points(sample_result, MSPI_TIMING_CONFIG_NUM_DEFAULT, &consecutive_length, &last_success_point);
    s_select_best_tuning_config(timing_config, consecutive_length, last_success_point, reference_data, is_flash);
}
#endif  //#if MSPI_TIMING_FLASH_NEEDS_TUNING || MSPI_TIMING_PSRAM_NEEDS_TUNING


/*------------------------------------------------------------------------------
 * FLASH Timing Tuning
 *----------------------------------------------------------------------------*/
#if MSPI_TIMING_FLASH_NEEDS_TUNING
void mspi_timing_flash_tuning(void)
{
    /**
     * set MSPI related regs to 20mhz configuration, to get reference data from FLASH
     * see detailed comments in this function (`mspi_timing_enter_low_speed_mode`)
     */
    mspi_timing_enter_low_speed_mode(true);

#if SOC_MEMSPI_TIMING_TUNING_BY_MSPI_DELAY
    mspi_tuning_cfg_drv_t drv = {
        .flash_tuning_type = MSPI_TIMING_TUNING_MSPI_DIN_DUMMY,
        .flash_init_mspi = mspi_timing_flash_init,
        .flash_tune_mspi = mspi_timing_config_flash_set_tuning_regs,
        .flash_read = mspi_timing_config_flash_read_data,
        .flash_select_best_tuning_config = mspi_timing_flash_select_best_tuning_config,
        .flash_set_best_tuning_config = mspi_timing_flash_set_best_tuning_config,
    };
    bool is_flash = true;
    s_register_config_driver(&drv, is_flash);

    //Disable the variable dummy mode when doing timing tuning
    mspi_timing_ll_enable_flash_variable_dummy(1, false);    //GD flash will read error in variable mode with 20MHz

    uint8_t reference_data[MSPI_TIMING_TEST_DATA_LEN] = {0};
    s_tuning_cfg_drv.flash_read(reference_data, MSPI_TIMING_FLASH_TEST_DATA_ADDR, sizeof(reference_data));

    mspi_timing_config_t timing_configs = {0};
    mspi_timing_get_flash_tuning_configs(&timing_configs);

#endif  //SOC_MEMSPI_TIMING_TUNING_BY_MSPI_DELAY

    s_do_tuning(reference_data, &timing_configs, true);

    mspi_timing_enter_high_speed_mode(true);
}
#else
void mspi_timing_flash_tuning(void)
{
    //Empty function for compatibility, therefore upper layer won't need to know that FLASH in which operation mode and frequency config needs to be tuned
}
#endif  //MSPI_TIMING_FLASH_NEEDS_TUNING


/*------------------------------------------------------------------------------
 * PSRAM Timing Tuning
 *----------------------------------------------------------------------------*/
#if MSPI_TIMING_PSRAM_NEEDS_TUNING
void mspi_timing_psram_tuning(void)
{
    /**
     * set MSPI related regs to 20mhz configuration, to write reference data to PSRAM
     * see detailed comments in this function (`mspi_timing_enter_low_speed_mode`)
     */
    mspi_timing_enter_low_speed_mode(true);

#if SOC_MEMSPI_TIMING_TUNING_BY_MSPI_DELAY
    // write data into psram, used to do timing tuning test.
    uint8_t reference_data[MSPI_TIMING_TEST_DATA_LEN];
    for (int i=0; i < MSPI_TIMING_TEST_DATA_LEN/4; i++) {
        ((uint32_t *)reference_data)[i] = 0xa5ff005a;
    }
    mspi_timing_config_psram_write_data(reference_data, MSPI_TIMING_PSRAM_TEST_DATA_ADDR, MSPI_TIMING_TEST_DATA_LEN);

    mspi_tuning_cfg_drv_t drv = {
        .psram_tuning_type = MSPI_TIMING_TUNING_MSPI_DIN_DUMMY,
        .psram_init_mspi = mspi_timing_psram_init,
        .psram_tune_mspi = mspi_timing_config_psram_set_tuning_regs,
        .psram_read = mspi_timing_config_psram_read_data,
        .psram_select_best_tuning_config = mspi_timing_psram_select_best_tuning_config,
        .psram_set_best_tuning_config = mspi_timing_psram_set_best_tuning_config,
    };
    bool is_flash = false;
    s_register_config_driver(&drv, is_flash);

    mspi_timing_config_t timing_configs = {0};
    mspi_timing_get_psram_tuning_configs(&timing_configs);
#endif //#if SOC_MEMSPI_TIMING_TUNING_BY_MSPI_DELAY

    //Disable the variable dummy mode when doing timing tuning
    mspi_timing_ll_enable_flash_variable_dummy(1, false);
    //Get required config, and set them to PSRAM related registers
    s_do_tuning(reference_data, &timing_configs, false);

    mspi_timing_enter_high_speed_mode(true);
}

#else
void mspi_timing_psram_tuning(void)
{
    //Empty function for compatibility, therefore upper layer won't need to know that FLASH in which operation mode and frequency config needs to be tuned
}
#endif  //MSPI_TIMING_PSRAM_NEEDS_TUNING


/*------------------------------------------------------------------------------
 * APIs to make SPI0 (and SPI1) FLASH work for high/low freq
 *----------------------------------------------------------------------------*/
void mspi_timing_enter_low_speed_mode(bool control_spi1)
{
#if SOC_MEMSPI_FLASH_CLK_SRC_IS_INDEPENDENT
    spimem_flash_ll_set_clock_source(MSPI_CLK_SRC_ROM_DEFAULT);
#endif  //SOC_MEMSPI_FLASH_CLK_SRC_IS_INDEPENDENT

#if SOC_SPI_MEM_SUPPORT_TIMING_TUNING
    /**
     * Here we are going to slow the SPI1 frequency to 20Mhz, so we need to set SPI1 din_num and din_mode regs.
     *
     * Because SPI0 and SPI1 share the din_num and din_mode regs, so if we clear SPI1 din_num and din_mode to
     * 0, if the SPI0 flash module clock is still in high freq, it may not work correctly.
     *
     * Therefore, here we need to slow both the SPI0 and SPI1 and related timing tuning regs to 20Mhz configuration.
     *
     * Currently we only need to change these clocks on chips with timing tuning
     * Should be extended to other no-timing-tuning chips if needed. e.g.:
     * we still need to turn down Flash / PSRAM clock speed at a certain period of time
     */
    mspi_timing_config_set_flash_clock(20, MSPI_TIMING_SPEED_MODE_LOW_PERF, control_spi1);
    mspi_timing_config_set_psram_clock(20, MSPI_TIMING_SPEED_MODE_LOW_PERF, control_spi1);
#endif  //#if SOC_SPI_MEM_SUPPORT_TIMING_TUNING

#if MSPI_TIMING_FLASH_NEEDS_TUNING || MSPI_TIMING_PSRAM_NEEDS_TUNING
    mspi_timing_flash_config_clear_tuning_regs(control_spi1);
    mspi_timing_psram_config_clear_tuning_regs(control_spi1);
#endif  //#if MSPI_TIMING_FLASH_NEEDS_TUNING || MSPI_TIMING_PSRAM_NEEDS_TUNING
}

/**
 * Set FLASH and PSRAM module clock, din_num, din_mode and extra dummy,
 * according to the configuration got from timing tuning function (`calculate_best_flash_tuning_config`).
 * iF control_spi1 == 1, will also update SPI1 timing registers. Should only be set to 1 when do tuning.
 *
 * This function should always be called after `mspi_timing_flash_tuning` or `calculate_best_flash_tuning_config`
 */
void mspi_timing_enter_high_speed_mode(bool control_spi1)
{
#if SOC_MEMSPI_FLASH_CLK_SRC_IS_INDEPENDENT
    spimem_flash_ll_set_clock_source(MSPI_CLK_SRC_DEFAULT);
#endif  //SOC_MEMSPI_FLASH_CLK_SRC_IS_INDEPENDENT

#if SOC_SPI_MEM_SUPPORT_TIMING_TUNING
    /**
     * Currently we only need to change these clocks on chips with timing tuning
     * Should be extended to other no-timing-tuning chips if needed. e.g.:
     * we still need to turn down Flash / PSRAM clock speed at a certain period of time
     */
    mspi_timing_config_set_flash_clock(FLASH_FREQUENCY_MHZ, MSPI_TIMING_SPEED_MODE_NORMAL_PERF, control_spi1);
#if CONFIG_SPIRAM
    mspi_timing_config_set_psram_clock(CONFIG_SPIRAM_SPEED, MSPI_TIMING_SPEED_MODE_NORMAL_PERF, control_spi1);
#endif  //#if CONFIG_SPIRAM
#endif  //#if SOC_SPI_MEM_SUPPORT_TIMING_TUNING

#if MSPI_TIMING_FLASH_NEEDS_TUNING || MSPI_TIMING_PSRAM_NEEDS_TUNING
    mspi_timing_flash_config_set_tuning_regs(control_spi1);
    mspi_timing_psram_config_set_tuning_regs(control_spi1);
#endif  //#if MSPI_TIMING_FLASH_NEEDS_TUNING || MSPI_TIMING_PSRAM_NEEDS_TUNING
}

void mspi_timing_change_speed_mode_cache_safe(bool switch_down)
{
    /**
     * If a no-cache-freeze-supported chip needs timing tuning, add a protection way:
     * - spinlock
     * - or other way
     *
     * for preventing concurrent from MSPI to external memory
     */
#if SOC_CACHE_FREEZE_SUPPORTED
    cache_hal_freeze(CACHE_TYPE_ALL);
#endif  //#if SOC_CACHE_FREEZE_SUPPORTED

    if (switch_down) {
        //enter MSPI low speed mode, extra delays should be removed
        mspi_timing_enter_low_speed_mode(false);
    } else {
        //enter MSPI high speed mode, extra delays should be considered
        mspi_timing_enter_high_speed_mode(false);
    }

#if SOC_CACHE_FREEZE_SUPPORTED
    cache_hal_unfreeze(CACHE_TYPE_ALL);
#endif  //#if SOC_CACHE_FREEZE_SUPPORTED
}

/*------------------------------------------------------------------------------
 * APIs to inform SPI1 Flash driver of necessary timing configurations
 *----------------------------------------------------------------------------*/
bool spi_timing_is_tuned(void)
{
#if MSPI_TIMING_FLASH_NEEDS_TUNING || MSPI_TIMING_PSRAM_NEEDS_TUNING
    return true;
#else
    return false;
#endif
}

#if MSPI_TIMING_FLASH_NEEDS_TUNING || MSPI_TIMING_PSRAM_NEEDS_TUNING
void spi_timing_get_flash_timing_param(spi_flash_hal_timing_config_t *out_timing_config)
{
    // Get clock configuration directly from system.
    out_timing_config->clock_config.spimem = mspi_timing_config_get_flash_clock_reg();

    // Get extra dummy length here. Therefore, no matter what freq, or mode.
    // If it needs tuning, it will return correct extra dummy len. If no tuning, it will return 0.

    out_timing_config->extra_dummy = mspi_timing_config_get_flash_extra_dummy();

    // Get CS setup/hold value here.
    mspi_timing_config_get_cs_timing(&out_timing_config->cs_setup, &out_timing_config->cs_hold);
}
#else
void spi_timing_get_flash_timing_param(spi_flash_hal_timing_config_t *out_timing_config)
{
    // This function shouldn't be called if timing tuning is not used.
    abort();
}
#endif // MSPI_TIMING_FLASH_NEEDS_TUNING || MSPI_TIMING_PSRAM_NEEDS_TUNING

/*------------------------------------------------------------------------------
 * Common settings
 *----------------------------------------------------------------------------*/
void mspi_timing_set_pin_drive_strength(void)
{
#if SOC_MEMSPI_TIMING_TUNING_BY_MSPI_DELAY
    //For now, set them all to 3. Need to check after QVL test results are out. TODO: IDF-3663
    //Set default pin drive
    mspi_timing_ll_set_all_pin_drive(0, 3);
#endif  //  #if SOC_MEMSPI_TIMING_TUNING_BY_MSPI_DELAY
}