Iterative Include SQL Simplification

[LTA-Thinking]

Description

Simplifies the SQL query generated when lots of iterative include statements are added to a query.

Related issues

Addresses User Story 130995

Testing

Tested the results returned before and after including the simplification on a known data set.

FHIR Team Checklist

• Update the title of the PR to be succinct and less than 65 characters
• Add a milestone to the PR for the sprint that it is merged (i.e. add S47)
• Tag the PR with the type of update: Bug, Build, Dependencies, Enhancement, New-Feature or Documentation
• Tag the PR with Open source, Azure API for FHIR (CosmosDB or common code) or Azure Healthcare APIs (SQL or common code) to specify where this change is intended to be released.
• Tag the PR with Schema Version backward compatible or Schema Version backward incompatible or Schema Version unchanged if this adds or updates Sql script which is/is not backward compatible with the code.
• CI is green before merge
• Review squash-merge requirements

Semver Change (docs)

Patch|Skip|Feature|Breaking (reason)

[LTA-Thinking]

/azp run

[azure-pipelines]

Azure Pipelines successfully started running 1 pipeline(s).

[brendankowitz]

Are we sure this is simpler than fixing the actual generator? Should this be a rewriter instead?

[LTA-Thinking]

I can take another look. Changing the generator looked more complicated.

[LTA-Thinking]

For the query:
GET https://{{hostname}}/DiagnosticReport?_include=DiagnosticReport:encounter&_include:iterate=Location:organization&_include:iterate=DiagnosticReport:based-on&_include:iterate=Encounter:location&_include:iterate=ServiceRequest:encounter

Original:

      DECLARE @p0 int = 11
      DECLARE @p1 int = 11
      DECLARE @p2 int = 100
      DECLARE @p3 int = 100
      DECLARE @p4 int = 100
      DECLARE @p5 int = 100

      SET STATISTICS IO ON;
      SET STATISTICS TIME ON;

      DECLARE @FilteredData AS TABLE (T1 smallint, Sid1 bigint, IsMatch bit, IsPartial bit, Row int)
      ;WITH
      cte0 AS
      (
          SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1
          FROM dbo.Resource
          WHERE IsHistory = 0
              AND IsDeleted = 0
              AND ResourceTypeId = 40
      )
      ,cte1 AS
      (
          SELECT row_number() OVER (ORDER BY T1 ASC, Sid1 ASC) AS Row, *
          FROM
          (
              SELECT DISTINCT TOP (@p0) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial
              FROM cte0
              ORDER BY T1 ASC, Sid1 ASC
          ) t
      )
      INSERT INTO @FilteredData SELECT T1, Sid1, IsMatch, IsPartial, Row FROM cte1
      ;WITH cte1 AS (SELECT * FROM @FilteredData)
      ,cte2 AS
      (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 204
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (40)
              AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 AND Row < @p1)
      )
      ,cte3 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte2
      )
      ,cte4 AS
      (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 404
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (40)
              AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
      )
      ,cte5 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte4
      )
      ,cte6 AS
      (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 204
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (124)
              AND EXISTS (SELECT * FROM cte5 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
      )
      ,cte7 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte6
      )
      ,cte8 AS
      (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 470
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (44)
              AND EXISTS (SELECT * FROM cte3 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
          )
          ,cte9 AS
          (
          SELECT DISTINCT TOP (@p2) T1, Sid1, IsMatch, CASE WHEN count_big(*) over() > @p3 THEN 1 ELSE 0 END AS IsPartial
          FROM cte8
          )
          ,cte10 AS
          (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 470
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (44)
              AND EXISTS (SELECT * FROM cte7 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
      )
      ,cte11 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte10
      )
      ,cte12 AS
      (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 770
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (71)
              AND EXISTS (SELECT * FROM cte9 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
          )
          ,cte13 AS
          (
          SELECT DISTINCT TOP (@p4) T1, Sid1, IsMatch, CASE WHEN count_big(*) over() > @p5 THEN 1 ELSE 0 END AS IsPartial
          FROM cte12
          )
          ,cte14 AS
          (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 770
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (71)
              AND EXISTS (SELECT * FROM cte11 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
      )
      ,cte15 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte14
      )
      ,cte16 AS
      (
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte1
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte3 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte3.Sid1 AND cte1.T1 = cte3.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte5 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte5.Sid1 AND cte1.T1 = cte5.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte7 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte7.Sid1 AND cte1.T1 = cte7.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte9 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte9.Sid1 AND cte1.T1 = cte9.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte11 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte11.Sid1 AND cte1.T1 = cte11.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte13 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte13.Sid1 AND cte1.T1 = cte13.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte15 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte15.Sid1 AND cte1.T1 = cte15.T1)
      )
      SELECT DISTINCT r.ResourceTypeId, r.ResourceId, r.Version, r.IsDeleted, r.ResourceSurrogateId, r.RequestMethod, CAST(IsMatch AS bit) AS IsMatch, CAST(IsPartial AS bit) AS IsPartial, r.IsRawResourceMetaSet, r.SearchParamHash, r.RawResource
      FROM dbo.Resource r
           JOIN cte16 ON r.ResourceTypeId = cte16.T1 AND r.ResourceSurrogateId = cte16.Sid1
      WHERE IsHistory = 0
          AND IsDeleted = 0
      ORDER BY IsMatch DESC, r.ResourceTypeId ASC, r.ResourceSurrogateId ASC

After change:

      DECLARE @p0 int = 11
      DECLARE @p1 int = 11
      DECLARE @p2 int = 100
      DECLARE @p3 int = 100
      DECLARE @p4 int = 100
      DECLARE @p5 int = 100

      SET STATISTICS IO ON;
      SET STATISTICS TIME ON;

      DECLARE @FilteredData AS TABLE (T1 smallint, Sid1 bigint, IsMatch bit, IsPartial bit, Row int)
      ;WITH
      cte0 AS
      (
          SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1
          FROM dbo.Resource
          WHERE IsHistory = 0
              AND IsDeleted = 0
              AND ResourceTypeId = 40
      )
      ,cte1 AS
      (
          SELECT row_number() OVER (ORDER BY T1 ASC, Sid1 ASC) AS Row, *
          FROM
          (
              SELECT DISTINCT TOP (@p0) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial
              FROM cte0
              ORDER BY T1 ASC, Sid1 ASC
          ) t
      )
      INSERT INTO @FilteredData SELECT T1, Sid1, IsMatch, IsPartial, Row FROM cte1
      ;WITH cte1 AS (SELECT * FROM @FilteredData)
      ,cte2 AS
      (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 204
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (40)
              AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 AND Row < @p1)
      )
      ,cte3 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte2
      )
      ,cte4 AS
      (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 404
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (40)
              AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
      )
      ,cte5 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte4
      )
      ,cte6 AS
      (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 204
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (124)
              AND EXISTS (SELECT * FROM cte5 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
      )
      ,cte7 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte6
      )
          ,cte10 AS
          (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 470
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (44)
              AND EXISTS (SELECT * FROM cte7 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 UNION SELECT * FROM cte3 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
      )
      ,cte11 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte10
      )
          ,cte14 AS
          (
          SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch
          FROM dbo.ReferenceSearchParam refSource
               JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId
          WHERE refSource.SearchParamId = 770
              AND refTarget.IsHistory = 0
              AND refTarget.IsDeleted = 0
              AND refSource.ResourceTypeId IN (71)
              AND EXISTS (SELECT * FROM cte11 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)
      )
      ,cte15 AS
      (
          SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial
          FROM cte14
      )
      ,cte16 AS
      (
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte1
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte3 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte3.Sid1 AND cte1.T1 = cte3.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte5 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte5.Sid1 AND cte1.T1 = cte5.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte7 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte7.Sid1 AND cte1.T1 = cte7.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte11 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte11.Sid1 AND cte1.T1 = cte11.T1)
          UNION ALL
          SELECT T1, Sid1, IsMatch, IsPartial
          FROM cte15 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte15.Sid1 AND cte1.T1 = cte15.T1)
      )
      SELECT DISTINCT r.ResourceTypeId, r.ResourceId, r.Version, r.IsDeleted, r.ResourceSurrogateId, r.RequestMethod, CAST(IsMatch AS bit) AS IsMatch, CAST(IsPartial AS bit) AS IsPartial, r.IsRawResourceMetaSet, r.SearchParamHash, r.RawResource
      FROM dbo.Resource r
           JOIN cte16 ON r.ResourceTypeId = cte16.T1 AND r.ResourceSurrogateId = cte16.Sid1
      WHERE IsHistory = 0
          AND IsDeleted = 0
      ORDER BY IsMatch DESC, r.ResourceTypeId ASC, r.ResourceSurrogateId ASC