Re: [PATCH v2 14/20] tools/testing/cxl: Define a fixed volatile configuration to parse

["Verma, Vishal L" <vishal.l.verma@intel.com>]

On Fri, 2023-02-10 at 01:06 -0800, Dan Williams wrote:
> Take two endpoints attached to the first switch on the first host-bridge
> in the cxl_test topology and define a pre-initialized region. This is a
> x2 interleave underneath a x1 CXL Window.
> 
> $ modprobe cxl_test
> $ # cxl list -Ru
> {
>   "region":"region3",
>   "resource":"0xf010000000",
>   "size":"512.00 MiB (536.87 MB)",
>   "interleave_ways":2,
>   "interleave_granularity":4096,
>   "decode_state":"commit"
> }
> 
> Tested-by: Fan Ni <fan.ni@samsung.com>
> Link: https://lore.kernel.org/r/167564541523.847146.12199636368812381475.stgit@dwillia2-xfh.jf.intel.com
> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
> ---
>  drivers/cxl/core/core.h      |    3 -
>  drivers/cxl/core/hdm.c       |    3 +
>  drivers/cxl/core/port.c      |    2 +
>  drivers/cxl/cxl.h            |    2 +
>  drivers/cxl/cxlmem.h         |    3 +
>  tools/testing/cxl/test/cxl.c |  147 +++++++++++++++++++++++++++++++++++++++---
>  6 files changed, 146 insertions(+), 14 deletions(-)

Looks good,

Reviewed-by: Vishal Verma <vishal.l.verma@intel.com>

> 
> diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
> index 5eb873da5a30..479f01da6d35 100644
> --- a/drivers/cxl/core/core.h
> +++ b/drivers/cxl/core/core.h
> @@ -57,9 +57,6 @@ resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled);
>  resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled);
>  extern struct rw_semaphore cxl_dpa_rwsem;
>  
> -bool is_switch_decoder(struct device *dev);
> -struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);
> -
>  int cxl_memdev_init(void);
>  void cxl_memdev_exit(void);
>  void cxl_mbox_init(void);
> diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
> index 8c29026a4b9d..80eccae6ba9e 100644
> --- a/drivers/cxl/core/hdm.c
> +++ b/drivers/cxl/core/hdm.c
> @@ -279,7 +279,7 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
>         return 0;
>  }
>  
> -static int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
> +int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
>                                 resource_size_t base, resource_size_t len,
>                                 resource_size_t skipped)
>  {
> @@ -295,6 +295,7 @@ static int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
>  
>         return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
>  }
> +EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL);
>  
>  resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
>  {
> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> index 59620528571a..b45d2796ef35 100644
> --- a/drivers/cxl/core/port.c
> +++ b/drivers/cxl/core/port.c
> @@ -458,6 +458,7 @@ bool is_switch_decoder(struct device *dev)
>  {
>         return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
>  }
> +EXPORT_SYMBOL_NS_GPL(is_switch_decoder, CXL);
>  
>  struct cxl_decoder *to_cxl_decoder(struct device *dev)
>  {
> @@ -485,6 +486,7 @@ struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
>                 return NULL;
>         return container_of(dev, struct cxl_switch_decoder, cxld.dev);
>  }
> +EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, CXL);
>  
>  static void cxl_ep_release(struct cxl_ep *ep)
>  {
> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> index c8ee4bb8cce6..2ac344235235 100644
> --- a/drivers/cxl/cxl.h
> +++ b/drivers/cxl/cxl.h
> @@ -653,8 +653,10 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
>  
>  struct cxl_decoder *to_cxl_decoder(struct device *dev);
>  struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev);
> +struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);
>  struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev);
>  bool is_root_decoder(struct device *dev);
> +bool is_switch_decoder(struct device *dev);
>  bool is_endpoint_decoder(struct device *dev);
>  struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
>                                                 unsigned int nr_targets,
> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> index c9da3c699a21..bf7d4c5c8612 100644
> --- a/drivers/cxl/cxlmem.h
> +++ b/drivers/cxl/cxlmem.h
> @@ -81,6 +81,9 @@ static inline bool is_cxl_endpoint(struct cxl_port *port)
>  }
>  
>  struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds);
> +int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
> +                        resource_size_t base, resource_size_t len,
> +                        resource_size_t skipped);
>  
>  static inline struct cxl_ep *cxl_ep_load(struct cxl_port *port,
>                                          struct cxl_memdev *cxlmd)
> diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c
> index 920bd969c554..5342f69d70d2 100644
> --- a/tools/testing/cxl/test/cxl.c
> +++ b/tools/testing/cxl/test/cxl.c
> @@ -703,6 +703,142 @@ static int mock_decoder_reset(struct cxl_decoder *cxld)
>         return 0;
>  }
>  
> +static void default_mock_decoder(struct cxl_decoder *cxld)
> +{
> +       cxld->hpa_range = (struct range){
> +               .start = 0,
> +               .end = -1,
> +       };
> +
> +       cxld->interleave_ways = 1;
> +       cxld->interleave_granularity = 256;
> +       cxld->target_type = CXL_DECODER_EXPANDER;
> +       cxld->commit = mock_decoder_commit;
> +       cxld->reset = mock_decoder_reset;
> +}
> +
> +static int first_decoder(struct device *dev, void *data)
> +{
> +       struct cxl_decoder *cxld;
> +
> +       if (!is_switch_decoder(dev))
> +               return 0;
> +       cxld = to_cxl_decoder(dev);
> +       if (cxld->id == 0)
> +               return 1;
> +       return 0;
> +}
> +
> +static void mock_init_hdm_decoder(struct cxl_decoder *cxld)
> +{
> +       struct acpi_cedt_cfmws *window = mock_cfmws[0];
> +       struct platform_device *pdev = NULL;
> +       struct cxl_endpoint_decoder *cxled;
> +       struct cxl_switch_decoder *cxlsd;
> +       struct cxl_port *port, *iter;
> +       const int size = SZ_512M;
> +       struct cxl_memdev *cxlmd;
> +       struct cxl_dport *dport;
> +       struct device *dev;
> +       bool hb0 = false;
> +       u64 base;
> +       int i;
> +
> +       if (is_endpoint_decoder(&cxld->dev)) {
> +               cxled = to_cxl_endpoint_decoder(&cxld->dev);
> +               cxlmd = cxled_to_memdev(cxled);
> +               WARN_ON(!dev_is_platform(cxlmd->dev.parent));
> +               pdev = to_platform_device(cxlmd->dev.parent);
> +
> +               /* check is endpoint is attach to host-bridge0 */
> +               port = cxled_to_port(cxled);
> +               do {
> +                       if (port->uport == &cxl_host_bridge[0]->dev) {
> +                               hb0 = true;
> +                               break;
> +                       }
> +                       if (is_cxl_port(port->dev.parent))
> +                               port = to_cxl_port(port->dev.parent);
> +                       else
> +                               port = NULL;
> +               } while (port);
> +               port = cxled_to_port(cxled);
> +       }
> +
> +       /*
> +        * The first decoder on the first 2 devices on the first switch
> +        * attached to host-bridge0 mock a fake / static RAM region. All
> +        * other decoders are default disabled. Given the round robin
> +        * assignment those devices are named cxl_mem.0, and cxl_mem.4.
> +        *
> +        * See 'cxl list -BMPu -m cxl_mem.0,cxl_mem.4'
> +        */
> +       if (!hb0 || pdev->id % 4 || pdev->id > 4 || cxld->id > 0) {
> +               default_mock_decoder(cxld);
> +               return;
> +       }
> +
> +       base = window->base_hpa;
> +       cxld->hpa_range = (struct range) {
> +               .start = base,
> +               .end = base + size - 1,
> +       };
> +
> +       cxld->interleave_ways = 2;
> +       eig_to_granularity(window->granularity, &cxld->interleave_granularity);
> +       cxld->target_type = CXL_DECODER_EXPANDER;
> +       cxld->flags = CXL_DECODER_F_ENABLE;
> +       cxled->state = CXL_DECODER_STATE_AUTO;
> +       port->commit_end = cxld->id;
> +       devm_cxl_dpa_reserve(cxled, 0, size / cxld->interleave_ways, 0);
> +       cxld->commit = mock_decoder_commit;
> +       cxld->reset = mock_decoder_reset;
> +
> +       /*
> +        * Now that endpoint decoder is set up, walk up the hierarchy
> +        * and setup the switch and root port decoders targeting @cxlmd.
> +        */
> +       iter = port;
> +       for (i = 0; i < 2; i++) {
> +               dport = iter->parent_dport;
> +               iter = dport->port;
> +               dev = device_find_child(&iter->dev, NULL, first_decoder);
> +               /*
> +                * Ancestor ports are guaranteed to be enumerated before
> +                * @port, and all ports have at least one decoder.
> +                */
> +               if (WARN_ON(!dev))
> +                       continue;
> +               cxlsd = to_cxl_switch_decoder(dev);
> +               if (i == 0) {
> +                       /* put cxl_mem.4 second in the decode order */
> +                       if (pdev->id == 4)
> +                               cxlsd->target[1] = dport;
> +                       else
> +                               cxlsd->target[0] = dport;
> +               } else
> +                       cxlsd->target[0] = dport;
> +               cxld = &cxlsd->cxld;
> +               cxld->target_type = CXL_DECODER_EXPANDER;
> +               cxld->flags = CXL_DECODER_F_ENABLE;
> +               iter->commit_end = 0;
> +               /*
> +                * Switch targets 2 endpoints, while host bridge targets
> +                * one root port
> +                */
> +               if (i == 0)
> +                       cxld->interleave_ways = 2;
> +               else
> +                       cxld->interleave_ways = 1;
> +               cxld->interleave_granularity = 256;
> +               cxld->hpa_range = (struct range) {
> +                       .start = base,
> +                       .end = base + size - 1,
> +               };
> +               put_device(dev);
> +       }
> +}
> +
>  static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)
>  {
>         struct cxl_port *port = cxlhdm->port;
> @@ -748,16 +884,7 @@ static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)
>                         cxld = &cxled->cxld;
>                 }
>  
> -               cxld->hpa_range = (struct range) {
> -                       .start = 0,
> -                       .end = -1,
> -               };
> -
> -               cxld->interleave_ways = min_not_zero(target_count, 1);
> -               cxld->interleave_granularity = SZ_4K;
> -               cxld->target_type = CXL_DECODER_EXPANDER;
> -               cxld->commit = mock_decoder_commit;
> -               cxld->reset = mock_decoder_reset;
> +               mock_init_hdm_decoder(cxld);
>  
>                 if (target_count) {
>                         rc = device_for_each_child(port->uport, &ctx,
>