Tech Report: Technologies total origins from crawl

definitions/output/reports/cwv_tech_technologies.js

@@ -6,19 +6,74 @@ publish('cwv_tech_technologies', {
   tags: ['crux_ready']
 }).query(ctx => `
 /* {"dataform_trigger": "report_cwv_tech_complete", "name": "technologies", "type": "dict"} */
+WITH pages AS (
+  SELECT DISTINCT
+    client,
+    root_page,
+    tech.technology
+  FROM ${ctx.ref('crawl', 'pages')},
+    UNNEST(technologies) AS tech
+  WHERE
+    date = '${pastMonth}'
+    ${constants.devRankFilter}
+),
+
+tech_origins AS (
+  SELECT
+    client,
+    technology,
+    COUNT(DISTINCT root_page) AS origins
+  FROM pages
+  GROUP BY
+    client,
+    technology
+),
+
+technologies AS (
+  SELECT
+    name AS technology,
+    description,
+    ARRAY_TO_STRING(categories, ', ') AS category,
+    categories AS category_obj,
+    NULL AS similar_technologies
+  FROM ${ctx.ref('wappalyzer', 'apps')}
+),
+
+total_pages AS (
+  SELECT
+    client,
+    COUNT(DISTINCT root_page) AS origins
+  FROM pages
+  GROUP BY client
+)
+
 SELECT
   client,
-  app AS technology,
+  technology,
   description,
   category,
-  SPLIT(category, ",") AS category_obj,
+  category_obj,
+  similar_technologies,
+  origins
+FROM tech_origins
+INNER JOIN technologies
+USING(technology)
+
+UNION ALL
+
+SELECT
+  client,
+  'ALL' AS technology,
+  NULL AS description,
+  ARRAY_TO_STRING(categories, ', ') AS category,
+  categories AS category_obj,
   NULL AS similar_technologies,
   origins
-FROM ${ctx.ref('core_web_vitals', 'technologies')}
-LEFT JOIN ${ctx.ref('wappalyzer', 'apps')}
-ON app = name
-WHERE date = '${pastMonth}' AND
-  geo = 'ALL' AND
-  rank = 'ALL'
-ORDER BY origins DESC
+FROM total_pages
+CROSS JOIN (
+  SELECT
+    ARRAY_AGG(DISTINCT category IGNORE NULLS ORDER BY category) AS categories
+  FROM technologies,
+    UNNEST(category_obj) AS category
+)
 `)