diff --git a/versions/.gitignore b/versions/.gitignore index 330372b12e..95c95bea87 100644 --- a/versions/.gitignore +++ b/versions/.gitignore @@ -7,3 +7,4 @@ logs/ # Rendered per-version cloud-init files cloud-init.*.sh +.local-server/ diff --git a/versions/apply-minvers.py b/versions/apply-minvers.py index 07abb53417..028f5917a0 100644 --- a/versions/apply-minvers.py +++ b/versions/apply-minvers.py @@ -31,6 +31,8 @@ def minver_lines(ds): # Version key mirroring run-version.sh's version_key: a bare build number is a 1.1. # snapshot; dotted versions use their components (missing -> 0). def vkey(v): + if v == "master": # the dev build is the newest + return (999999, 0, 0, 0) if re.fullmatch(r"\d+", v): return (1, 1, int(v), 0) parts = v.split(".") diff --git a/versions/create/create.sh b/versions/create/create.sh index 6ce3803aa4..c3e5dbda59 100755 --- a/versions/create/create.sh +++ b/versions/create/create.sh @@ -30,6 +30,7 @@ TABLE="${3:?usage: create.sh }" # on supports it. Older 1.1.x builds get the legacy positional engine. new_syntax() { local major minor patch + [ "$VERSION" = "master" ] && return 0 # dev build: modern custom-partitioning syntax IFS=. read -r major minor patch _ <<<"$VERSION" if [[ "$VERSION" =~ ^[0-9]+$ ]]; then # Bare build number (earliest 2016 releases, e.g. 53973..54011): custom diff --git a/versions/data.generated.js b/versions/data.generated.js index a60515fef2..ccb635bbe4 100644 --- a/versions/data.generated.js +++ b/versions/data.generated.js @@ -197,7 +197,7 @@ const datasets = [ "SELECT c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number, amt, profit FROM ( SELECT ss_ticket_number, ss_customer_sk, ca_city AS bought_city, sum(ss_coupon_amt) AS amt, sum(ss_net_profit) AS profit FROM store_sales, date_dim, store, household_demographics, customer_address WHERE (store_sales.ss_sold_date_sk = date_dim.d_date_sk) AND (store_sales.ss_store_sk = store.s_store_sk) AND (store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk) AND (store_sales.ss_addr_sk = customer_address.ca_address_sk) AND ((household_demographics.hd_dep_count = 4) OR (household_demographics.hd_vehicle_count = 3)) AND (date_dim.d_dow IN (6, 0)) AND (date_dim.d_year IN (1999, 1999 + 1, 1999 + 2)) AND (store.s_city IN ('Fairview', 'Midway', 'Fairview', 'Fairview', 'Fairview')) GROUP BY ss_ticket_number, ss_customer_sk, ss_addr_sk, ca_city ) AS dn, customer, customer_address AS current_addr WHERE (ss_customer_sk = c_customer_sk) AND (customer.c_current_addr_sk = current_addr.ca_address_sk) AND (current_addr.ca_city <> bought_city) ORDER BY c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number LIMIT 100", "WITH v1 AS ( SELECT i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) AS sum_sales, avg(sum(ss_sales_price)) OVER (PARTITION BY i_category, i_brand, s_store_name, s_company_name, d_year) AS avg_monthly_sales, rank() OVER (PARTITION BY i_category, i_brand, s_store_name, s_company_name ORDER BY d_year, d_moy) AS rn FROM item, store_sales, date_dim, store WHERE (ss_item_sk = i_item_sk) AND (ss_sold_date_sk = d_date_sk) AND (ss_store_sk = s_store_sk) AND ( (d_year = 1999) OR ((d_year = 1999 - 1) AND (d_moy = 12)) OR ((d_year = 1999 + 1) AND (d_moy = 1)) ) GROUP BY i_category, i_brand, s_store_name, s_company_name, d_year, d_moy ), v2 AS ( SELECT v1.i_category, v1.i_brand, v1.s_store_name, v1.s_company_name, v1.d_year, v1.d_moy, v1.avg_monthly_sales, v1.sum_sales, v1_lag.sum_sales AS psum, v1_lead.sum_sales AS nsum FROM v1, v1 AS v1_lag, v1 AS v1_lead WHERE (v1.i_category = v1_lag.i_category) AND (v1.i_category = v1_lead.i_category) AND (v1.i_brand = v1_lag.i_brand) AND (v1.i_brand = v1_lead.i_brand) AND (v1.s_store_name = v1_lag.s_store_name) AND (v1.s_store_name = v1_lead.s_store_name) AND (v1.s_company_name = v1_lag.s_company_name) AND (v1.s_company_name = v1_lead.s_company_name) AND (v1.rn = v1_lag.rn + 1) AND (v1.rn = v1_lead.rn - 1) ) SELECT * FROM v2 WHERE (v1.d_year = 1999) AND (v1.avg_monthly_sales > 0) AND (CASE WHEN v1.avg_monthly_sales > 0 THEN abs(v1.sum_sales - v1.avg_monthly_sales) / v1.avg_monthly_sales ELSE NULL END > 0.1) ORDER BY v1.sum_sales - v1.avg_monthly_sales, v1.s_store_name LIMIT 100", "SELECT sum(ss_quantity) FROM store_sales, store, customer_demographics, customer_address, date_dim WHERE (s_store_sk = ss_store_sk) AND (ss_sold_date_sk = d_date_sk) AND (d_year = 2000) AND ( ( (cd_demo_sk = ss_cdemo_sk) AND (cd_marital_status = 'M') AND (cd_education_status = '4 yr Degree') AND (ss_sales_price BETWEEN 100.00 AND 150.00) ) OR ( (cd_demo_sk = ss_cdemo_sk) AND (cd_marital_status = 'D') AND (cd_education_status = '2 yr Degree') AND (ss_sales_price BETWEEN 50.00 AND 100.00) ) OR ( (cd_demo_sk = ss_cdemo_sk) AND (cd_marital_status = 'S') AND (cd_education_status = 'College') AND (ss_sales_price BETWEEN 150.00 AND 200.00) ) ) AND ( ( (ss_addr_sk = ca_address_sk) AND (ca_country = 'United States') AND (ca_state IN ('CO', 'OH', 'TX')) AND (ss_net_profit BETWEEN 0 AND 2000) ) OR ( (ss_addr_sk = ca_address_sk) AND (ca_country = 'United States') AND (ca_state IN ('OR', 'MN', 'KY')) AND (ss_net_profit BETWEEN 150 AND 3000) ) OR ( (ss_addr_sk = ca_address_sk) AND (ca_country = 'United States') AND (ca_state IN ('VA', 'CA', 'MS')) AND (ss_net_profit BETWEEN 50 AND 25000) ) )", - "SELECT channel, item, return_ratio, return_rank, currency_rank FROM ( SELECT 'web' AS channel, web.item, web.return_ratio, web.return_rank, web.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT ws.ws_item_sk AS item, (CAST(sum(coalesce(wr.wr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(ws.ws_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(wr.wr_return_amt, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(ws.ws_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM web_sales AS ws LEFT OUTER JOIN web_returns AS wr ON (ws.ws_order_number = wr.wr_order_number) AND (ws.ws_item_sk = wr.wr_item_sk), date_dim WHERE (wr.wr_return_amt > 10000) AND (ws.ws_net_profit > 1) AND (ws.ws_net_paid > 0) AND (ws.ws_quantity > 0) AND (ws_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY ws.ws_item_sk ) AS in_web ) AS web WHERE (web.return_rank <= 10) OR (web.currency_rank <= 10) UNION SELECT 'catalog' AS channel, catalog.item, catalog.return_ratio, catalog.return_rank, catalog.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT cs.cs_item_sk AS item, (CAST(sum(coalesce(cr.cr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(cs.cs_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(cr.cr_return_amount, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(cs.cs_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM catalog_sales AS cs LEFT OUTER JOIN catalog_returns AS cr ON (cs.cs_order_number = cr.cr_order_number) AND (cs.cs_item_sk = cr.cr_item_sk), date_dim WHERE (cr.cr_return_amount > 10000) AND (cs.cs_net_profit > 1) AND (cs.cs_net_paid > 0) AND (cs.cs_quantity > 0) AND (cs_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY cs.cs_item_sk ) AS in_cat ) AS catalog WHERE (catalog.return_rank <= 10) OR (catalog.currency_rank <= 10) UNION SELECT 'store' AS channel, store.item, store.return_ratio, store.return_rank, store.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT sts.ss_item_sk AS item, (CAST(sum(coalesce(sr.sr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(sts.ss_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(sr.sr_return_amt, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM store_sales AS sts LEFT OUTER JOIN store_returns AS sr ON (sts.ss_ticket_number = sr.sr_ticket_number) AND (sts.ss_item_sk = sr.sr_item_sk), date_dim WHERE (sr.sr_return_amt > 10000) AND (sts.ss_net_profit > 1) AND (sts.ss_net_paid > 0) AND (sts.ss_quantity > 0) AND (ss_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY sts.ss_item_sk ) AS in_store ) AS store WHERE (store.return_rank <= 10) OR (store.currency_rank <= 10) ) ORDER BY 1, 4, 5, 2 LIMIT 100", + "SELECT channel, item, return_ratio, return_rank, currency_rank FROM ( SELECT 'web' AS channel, web.item, web.return_ratio, web.return_rank, web.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT ws.ws_item_sk AS item, (CAST(sum(coalesce(wr.wr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(ws.ws_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(wr.wr_return_amt, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(ws.ws_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM web_sales AS ws LEFT OUTER JOIN web_returns AS wr ON (ws.ws_order_number = wr.wr_order_number) AND (ws.ws_item_sk = wr.wr_item_sk), date_dim WHERE (wr.wr_return_amt > 10000) AND (ws.ws_net_profit > 1) AND (ws.ws_net_paid > 0) AND (ws.ws_quantity > 0) AND (ws_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY ws.ws_item_sk ) AS in_web ) AS web WHERE (web.return_rank <= 10) OR (web.currency_rank <= 10) UNION DISTINCT SELECT 'catalog' AS channel, catalog.item, catalog.return_ratio, catalog.return_rank, catalog.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT cs.cs_item_sk AS item, (CAST(sum(coalesce(cr.cr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(cs.cs_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(cr.cr_return_amount, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(cs.cs_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM catalog_sales AS cs LEFT OUTER JOIN catalog_returns AS cr ON (cs.cs_order_number = cr.cr_order_number) AND (cs.cs_item_sk = cr.cr_item_sk), date_dim WHERE (cr.cr_return_amount > 10000) AND (cs.cs_net_profit > 1) AND (cs.cs_net_paid > 0) AND (cs.cs_quantity > 0) AND (cs_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY cs.cs_item_sk ) AS in_cat ) AS catalog WHERE (catalog.return_rank <= 10) OR (catalog.currency_rank <= 10) UNION DISTINCT SELECT 'store' AS channel, store.item, store.return_ratio, store.return_rank, store.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT sts.ss_item_sk AS item, (CAST(sum(coalesce(sr.sr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(sts.ss_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(sr.sr_return_amt, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM store_sales AS sts LEFT OUTER JOIN store_returns AS sr ON (sts.ss_ticket_number = sr.sr_ticket_number) AND (sts.ss_item_sk = sr.sr_item_sk), date_dim WHERE (sr.sr_return_amt > 10000) AND (sts.ss_net_profit > 1) AND (sts.ss_net_paid > 0) AND (sts.ss_quantity > 0) AND (ss_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY sts.ss_item_sk ) AS in_store ) AS store WHERE (store.return_rank <= 10) OR (store.currency_rank <= 10) ) ORDER BY 1, 4, 5, 2 LIMIT 100", "SELECT s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip, sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk <= 30) THEN 1 ELSE 0 END) AS \"30 days\", sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk > 30) AND (sr_returned_date_sk - ss_sold_date_sk <= 60) THEN 1 ELSE 0 END) AS \"31-60 days\", sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk > 60) AND (sr_returned_date_sk - ss_sold_date_sk <= 90) THEN 1 ELSE 0 END) AS \"61-90 days\", sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk > 90) AND (sr_returned_date_sk - ss_sold_date_sk <= 120) THEN 1 ELSE 0 END) AS \"91-120 days\", sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk > 120) THEN 1 ELSE 0 END) AS \">120 days\" FROM store_sales, store_returns, store, date_dim AS d1, date_dim AS d2 WHERE (d2.d_year = 2001) AND (d2.d_moy = 8) AND (ss_ticket_number = sr_ticket_number) AND (ss_item_sk = sr_item_sk) AND (ss_sold_date_sk = d1.d_date_sk) AND (sr_returned_date_sk = d2.d_date_sk) AND (ss_customer_sk = sr_customer_sk) AND (ss_store_sk = s_store_sk) GROUP BY s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip ORDER BY s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip LIMIT 100", "WITH web_v1 AS ( SELECT ws_item_sk AS item_sk, d_date, sum(sum(ws_sales_price)) OVER (PARTITION BY ws_item_sk ORDER BY d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cume_sales FROM web_sales, date_dim WHERE (ws_sold_date_sk = d_date_sk) AND (d_month_seq BETWEEN 1200 AND 1200 + 11) AND (ws_item_sk IS NOT NULL) GROUP BY ws_item_sk, d_date ), store_v1 AS ( SELECT ss_item_sk AS item_sk, d_date, sum(sum(ss_sales_price)) OVER (PARTITION BY ss_item_sk ORDER BY d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cume_sales FROM store_sales, date_dim WHERE (ss_sold_date_sk = d_date_sk) AND (d_month_seq BETWEEN 1200 AND 1200 + 11) AND (ss_item_sk IS NOT NULL) GROUP BY ss_item_sk, d_date ) SELECT * FROM ( SELECT item_sk, d_date, web_sales, store_sales, max(web_sales) OVER (PARTITION BY item_sk ORDER BY d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS web_cumulative, max(store_sales) OVER (PARTITION BY item_sk ORDER BY d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS store_cumulative FROM ( SELECT CASE WHEN web.item_sk IS NOT NULL THEN web.item_sk ELSE store.item_sk END AS item_sk, CASE WHEN web.d_date IS NOT NULL THEN web.d_date ELSE store.d_date END AS d_date, web.cume_sales AS web_sales, store.cume_sales AS store_sales FROM web_v1 AS web FULL OUTER JOIN store_v1 AS store ON (web.item_sk = store.item_sk) AND (web.d_date = store.d_date) ) AS x ) AS y WHERE (web_cumulative > store_cumulative) ORDER BY item_sk, d_date LIMIT 100", "SELECT dt.d_year, item.i_brand_id AS brand_id, item.i_brand AS brand, sum(ss_ext_sales_price) AS ext_price FROM date_dim AS dt, store_sales, item WHERE (dt.d_date_sk = store_sales.ss_sold_date_sk) AND (store_sales.ss_item_sk = item.i_item_sk) AND (item.i_manager_id = 1) AND (dt.d_moy = 11) AND (dt.d_year = 2000) GROUP BY dt.d_year, item.i_brand, item.i_brand_id ORDER BY dt.d_year, ext_price DESC, brand_id LIMIT 100", @@ -223,7 +223,7 @@ const datasets = [ "SELECT i_item_desc, w_warehouse_name, d1.d_week_seq, sum(multiIf(p_promo_sk IS NULL, 1, 0)) AS no_promo, sum(multiIf(p_promo_sk IS NOT NULL, 1, 0)) AS promo, count(*) AS total_cnt FROM catalog_sales INNER JOIN inventory ON cs_item_sk = inv_item_sk INNER JOIN warehouse ON w_warehouse_sk = inv_warehouse_sk INNER JOIN item ON i_item_sk = cs_item_sk INNER JOIN customer_demographics ON cs_bill_cdemo_sk = cd_demo_sk INNER JOIN household_demographics ON cs_bill_hdemo_sk = hd_demo_sk INNER JOIN date_dim AS d1 ON cs_sold_date_sk = d1.d_date_sk INNER JOIN date_dim AS d2 ON inv_date_sk = d2.d_date_sk INNER JOIN date_dim AS d3 ON cs_ship_date_sk = d3.d_date_sk LEFT JOIN promotion ON cs_promo_sk = p_promo_sk LEFT JOIN catalog_returns ON (cr_item_sk = cs_item_sk) AND (cr_order_number = cs_order_number) WHERE (d1.d_week_seq = d2.d_week_seq) AND (inv_quantity_on_hand < cs_quantity) AND (d3.d_date > (d1.d_date + 5)) AND (hd_buy_potential = '>10000') AND (d1.d_year = 1999) AND (cd_marital_status = 'D') GROUP BY i_item_desc, w_warehouse_name, d_week_seq ORDER BY total_cnt DESC, i_item_desc, w_warehouse_name, d_week_seq LIMIT 100", "SELECT c_last_name, c_first_name, c_salutation, c_preferred_cust_flag, ss_ticket_number, cnt FROM ( SELECT ss_ticket_number, ss_customer_sk, count(*) AS cnt FROM store_sales, date_dim, store, household_demographics WHERE (store_sales.ss_sold_date_sk = date_dim.d_date_sk) AND (store_sales.ss_store_sk = store.s_store_sk) AND (store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk) AND ((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 2)) AND ((household_demographics.hd_buy_potential = '>10000') OR (household_demographics.hd_buy_potential = 'Unknown')) AND (household_demographics.hd_vehicle_count > 0) AND (multiIf(household_demographics.hd_vehicle_count > 0, household_demographics.hd_dep_count / household_demographics.hd_vehicle_count, NULL) > 1) AND (date_dim.d_year IN (1999, 1999 + 1, 1999 + 2)) AND (store.s_county IN ('Williamson County', 'Franklin Parish', 'Bronx County', 'Orange County')) GROUP BY ss_ticket_number, ss_customer_sk ) AS dj, customer WHERE (ss_customer_sk = c_customer_sk) AND ((cnt >= 1) AND (cnt <= 5)) ORDER BY cnt DESC, c_last_name", "WITH year_total AS ( SELECT c_customer_id AS customer_id, c_first_name AS customer_first_name, c_last_name AS customer_last_name, d_year AS year, sum(ss_net_paid) AS year_total, 's' AS sale_type FROM customer, store_sales, date_dim WHERE (c_customer_sk = ss_customer_sk) AND (ss_sold_date_sk = d_date_sk) AND (d_year IN (2001, 2001 + 1)) GROUP BY c_customer_id, c_first_name, c_last_name, d_year UNION ALL SELECT c_customer_id AS customer_id, c_first_name AS customer_first_name, c_last_name AS customer_last_name, d_year AS year, sum(ws_net_paid) AS year_total, 'w' AS sale_type FROM customer, web_sales, date_dim WHERE (c_customer_sk = ws_bill_customer_sk) AND (ws_sold_date_sk = d_date_sk) AND (d_year IN (2001, 2001 + 1)) GROUP BY c_customer_id, c_first_name, c_last_name, d_year ) SELECT t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name FROM year_total AS t_s_firstyear, year_total AS t_s_secyear, year_total AS t_w_firstyear, year_total AS t_w_secyear WHERE (t_s_secyear.customer_id = t_s_firstyear.customer_id) AND (t_s_firstyear.customer_id = t_w_secyear.customer_id) AND (t_s_firstyear.customer_id = t_w_firstyear.customer_id) AND (t_s_firstyear.sale_type = 's') AND (t_w_firstyear.sale_type = 'w') AND (t_s_secyear.sale_type = 's') AND (t_w_secyear.sale_type = 'w') AND (t_s_firstyear.year = 2001) AND (t_s_secyear.year = (2001 + 1)) AND (t_w_firstyear.year = 2001) AND (t_w_secyear.year = (2001 + 1)) AND (t_s_firstyear.year_total > 0) AND (t_w_firstyear.year_total > 0) AND (multiIf(t_w_firstyear.year_total > 0, t_w_secyear.year_total / t_w_firstyear.year_total, NULL) > multiIf(t_s_firstyear.year_total > 0, t_s_secyear.year_total / t_s_firstyear.year_total, NULL)) ORDER BY 1, 2, 3 LIMIT 100", - "WITH all_sales AS ( SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, SUM(sales_cnt) AS sales_cnt, SUM(sales_amt) AS sales_amt FROM ( SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, cs_quantity - COALESCE(cr_return_quantity, 0) AS sales_cnt, cs_ext_sales_price - COALESCE(cr_return_amount, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM catalog_sales INNER JOIN item ON i_item_sk = cs_item_sk INNER JOIN date_dim ON d_date_sk = cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number = cr_order_number) AND (cs_item_sk = cr_item_sk) WHERE (i_category = 'Books') UNION SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, ss_quantity - COALESCE(sr_return_quantity, 0) AS sales_cnt, ss_ext_sales_price - COALESCE(sr_return_amt, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM store_sales INNER JOIN item ON i_item_sk = ss_item_sk INNER JOIN date_dim ON d_date_sk = ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number = sr_ticket_number) AND (ss_item_sk = sr_item_sk) WHERE (i_category = 'Books') UNION SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, ws_quantity - COALESCE(wr_return_quantity, 0) AS sales_cnt, ws_ext_sales_price - COALESCE(wr_return_amt, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM web_sales INNER JOIN item ON i_item_sk = ws_item_sk INNER JOIN date_dim ON d_date_sk = ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number = wr_order_number) AND (ws_item_sk = wr_item_sk) WHERE (i_category = 'Books') ) AS sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id ) SELECT prev_yr.d_year AS prev_year, curr_yr.d_year AS year, curr_yr.i_brand_id, curr_yr.i_class_id, curr_yr.i_category_id, curr_yr.i_manufact_id, prev_yr.sales_cnt AS prev_yr_cnt, curr_yr.sales_cnt AS curr_yr_cnt, curr_yr.sales_cnt - prev_yr.sales_cnt AS sales_cnt_diff, curr_yr.sales_amt - prev_yr.sales_amt AS sales_amt_diff FROM all_sales AS curr_yr, all_sales AS prev_yr WHERE (curr_yr.i_brand_id = prev_yr.i_brand_id) AND (curr_yr.i_class_id = prev_yr.i_class_id) AND (curr_yr.i_category_id = prev_yr.i_category_id) AND (curr_yr.i_manufact_id = prev_yr.i_manufact_id) AND (curr_yr.d_year = 2002) AND (prev_yr.d_year = (2002 - 1)) AND ((CAST(curr_yr.sales_cnt, 'DECIMAL(17, 2)') / CAST(prev_yr.sales_cnt, 'DECIMAL(17, 2)')) < 0.9) ORDER BY sales_cnt_diff, sales_amt_diff LIMIT 100", + "WITH all_sales AS ( SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, SUM(sales_cnt) AS sales_cnt, SUM(sales_amt) AS sales_amt FROM ( SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, cs_quantity - COALESCE(cr_return_quantity, 0) AS sales_cnt, cs_ext_sales_price - COALESCE(cr_return_amount, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM catalog_sales INNER JOIN item ON i_item_sk = cs_item_sk INNER JOIN date_dim ON d_date_sk = cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number = cr_order_number) AND (cs_item_sk = cr_item_sk) WHERE (i_category = 'Books') UNION DISTINCT SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, ss_quantity - COALESCE(sr_return_quantity, 0) AS sales_cnt, ss_ext_sales_price - COALESCE(sr_return_amt, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM store_sales INNER JOIN item ON i_item_sk = ss_item_sk INNER JOIN date_dim ON d_date_sk = ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number = sr_ticket_number) AND (ss_item_sk = sr_item_sk) WHERE (i_category = 'Books') UNION DISTINCT SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, ws_quantity - COALESCE(wr_return_quantity, 0) AS sales_cnt, ws_ext_sales_price - COALESCE(wr_return_amt, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM web_sales INNER JOIN item ON i_item_sk = ws_item_sk INNER JOIN date_dim ON d_date_sk = ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number = wr_order_number) AND (ws_item_sk = wr_item_sk) WHERE (i_category = 'Books') ) AS sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id ) SELECT prev_yr.d_year AS prev_year, curr_yr.d_year AS year, curr_yr.i_brand_id, curr_yr.i_class_id, curr_yr.i_category_id, curr_yr.i_manufact_id, prev_yr.sales_cnt AS prev_yr_cnt, curr_yr.sales_cnt AS curr_yr_cnt, curr_yr.sales_cnt - prev_yr.sales_cnt AS sales_cnt_diff, curr_yr.sales_amt - prev_yr.sales_amt AS sales_amt_diff FROM all_sales AS curr_yr, all_sales AS prev_yr WHERE (curr_yr.i_brand_id = prev_yr.i_brand_id) AND (curr_yr.i_class_id = prev_yr.i_class_id) AND (curr_yr.i_category_id = prev_yr.i_category_id) AND (curr_yr.i_manufact_id = prev_yr.i_manufact_id) AND (curr_yr.d_year = 2002) AND (prev_yr.d_year = (2002 - 1)) AND ((CAST(curr_yr.sales_cnt, 'DECIMAL(17, 2)') / CAST(prev_yr.sales_cnt, 'DECIMAL(17, 2)')) < 0.9) ORDER BY sales_cnt_diff, sales_amt_diff LIMIT 100", "SELECT channel, col_name, d_year, d_qoy, i_category, COUNT(*) AS sales_cnt, SUM(ext_sales_price) AS sales_amt FROM ( SELECT 'store' AS channel, 'ss_store_sk' AS col_name, d_year, d_qoy, i_category, ss_ext_sales_price AS ext_sales_price FROM store_sales, item, date_dim WHERE (ss_store_sk IS NULL) AND (ss_sold_date_sk = d_date_sk) AND (ss_item_sk = i_item_sk) UNION ALL SELECT 'web' AS channel, 'ws_ship_customer_sk' AS col_name, d_year, d_qoy, i_category, ws_ext_sales_price AS ext_sales_price FROM web_sales, item, date_dim WHERE (ws_ship_customer_sk IS NULL) AND (ws_sold_date_sk = d_date_sk) AND (ws_item_sk = i_item_sk) UNION ALL SELECT 'catalog' AS channel, 'cs_ship_addr_sk' AS col_name, d_year, d_qoy, i_category, cs_ext_sales_price AS ext_sales_price FROM catalog_sales, item, date_dim WHERE (cs_ship_addr_sk IS NULL) AND (cs_sold_date_sk = d_date_sk) AND (cs_item_sk = i_item_sk) ) AS foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category LIMIT 100", "WITH ss AS ( SELECT s_store_sk, sum(ss_ext_sales_price) AS sales, sum(ss_net_profit) AS profit FROM store_sales, date_dim, store WHERE (ss_sold_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) AND (ss_store_sk = s_store_sk) GROUP BY s_store_sk ), sr AS ( SELECT s_store_sk, sum(sr_return_amt) AS returns, sum(sr_net_loss) AS profit_loss FROM store_returns, date_dim, store WHERE (sr_returned_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) AND (sr_store_sk = s_store_sk) GROUP BY s_store_sk ), cs AS ( SELECT cs_call_center_sk, sum(cs_ext_sales_price) AS sales, sum(cs_net_profit) AS profit FROM catalog_sales, date_dim WHERE (cs_sold_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) GROUP BY cs_call_center_sk ), cr AS ( SELECT cr_call_center_sk, sum(cr_return_amount) AS returns, sum(cr_net_loss) AS profit_loss FROM catalog_returns, date_dim WHERE (cr_returned_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) GROUP BY cr_call_center_sk ), ws AS ( SELECT wp_web_page_sk, sum(ws_ext_sales_price) AS sales, sum(ws_net_profit) AS profit FROM web_sales, date_dim, web_page WHERE (ws_sold_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) AND (ws_web_page_sk = wp_web_page_sk) GROUP BY wp_web_page_sk ), wr AS ( SELECT wp_web_page_sk, sum(wr_return_amt) AS returns, sum(wr_net_loss) AS profit_loss FROM web_returns, date_dim, web_page WHERE (wr_returned_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) AND (wr_web_page_sk = wp_web_page_sk) GROUP BY wp_web_page_sk ) SELECT channel, id, sum(sales) AS sales, sum(returns) AS returns, sum(profit) AS profit FROM ( SELECT 'store channel' AS channel, ss.s_store_sk AS id, sales, coalesce(returns, 0) AS returns, profit - coalesce(profit_loss, 0) AS profit FROM ss LEFT JOIN sr ON ss.s_store_sk = sr.s_store_sk UNION ALL SELECT 'catalog channel' AS channel, cs_call_center_sk AS id, sales, returns, profit - profit_loss AS profit FROM cs, cr UNION ALL SELECT 'web channel' AS channel, ws.wp_web_page_sk AS id, sales, coalesce(returns, 0) AS returns, profit - coalesce(profit_loss, 0) AS profit FROM ws LEFT JOIN wr ON ws.wp_web_page_sk = wr.wp_web_page_sk ) AS x GROUP BY channel, id WITH ROLLUP ORDER BY channel, id LIMIT 100", "WITH ws AS ( SELECT d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk AS ws_customer_sk, sum(ws_quantity) AS ws_qty, sum(ws_wholesale_cost) AS ws_wc, sum(ws_sales_price) AS ws_sp FROM web_sales LEFT JOIN web_returns ON (wr_order_number = ws_order_number) AND (ws_item_sk = wr_item_sk) INNER JOIN date_dim ON ws_sold_date_sk = d_date_sk WHERE wr_order_number IS NULL GROUP BY d_year, ws_item_sk, ws_bill_customer_sk ), cs AS ( SELECT d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk AS cs_customer_sk, sum(cs_quantity) AS cs_qty, sum(cs_wholesale_cost) AS cs_wc, sum(cs_sales_price) AS cs_sp FROM catalog_sales LEFT JOIN catalog_returns ON (cr_order_number = cs_order_number) AND (cs_item_sk = cr_item_sk) INNER JOIN date_dim ON cs_sold_date_sk = d_date_sk WHERE cr_order_number IS NULL GROUP BY d_year, cs_item_sk, cs_bill_customer_sk ), ss AS ( SELECT d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) AS ss_qty, sum(ss_wholesale_cost) AS ss_wc, sum(ss_sales_price) AS ss_sp FROM store_sales LEFT JOIN store_returns ON (sr_ticket_number = ss_ticket_number) AND (ss_item_sk = sr_item_sk) INNER JOIN date_dim ON ss_sold_date_sk = d_date_sk WHERE sr_ticket_number IS NULL GROUP BY d_year, ss_item_sk, ss_customer_sk ) SELECT ss_sold_year, ss_item_sk, ss_customer_sk, round(ss_qty / (coalesce(ws_qty, 0) + coalesce(cs_qty, 0)), 2) AS ratio, ss_qty AS store_qty, ss_wc AS store_wholesale_cost, ss_sp AS store_sales_price, coalesce(ws_qty, 0) + coalesce(cs_qty, 0) AS other_chan_qty, coalesce(ws_wc, 0) + coalesce(cs_wc, 0) AS other_chan_wholesale_cost, coalesce(ws_sp, 0) + coalesce(cs_sp, 0) AS other_chan_sales_price FROM ss LEFT JOIN ws ON (ws_sold_year = ss_sold_year) AND (ws_item_sk = ss_item_sk) AND (ws_customer_sk = ss_customer_sk) LEFT JOIN cs ON (cs_sold_year = ss_sold_year) AND (cs_item_sk = ss_item_sk) AND (cs_customer_sk = ss_customer_sk) WHERE ((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)) AND (ss_sold_year = 2000) ORDER BY ss_sold_year, ss_item_sk, ss_customer_sk, ss_qty DESC, ss_wc DESC, ss_sp DESC, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio LIMIT 100", diff --git a/versions/fetch-results.sh b/versions/fetch-results.sh index 45015437e7..3c41d44bfc 100755 --- a/versions/fetch-results.sh +++ b/versions/fetch-results.sh @@ -98,13 +98,13 @@ echo "wrote $(ls results/*.json 2>/dev/null | wc -l) result files" >&2 declare -A best for f in results/*.json; do v="$(basename "${f}" .json)" - case "${v}" in [0-9]*.[0-9]*) continue ;; esac # already has a dotted prefix + case "${v}" in [0-9]*.[0-9]*|master) continue ;; esac # dotted prefix, or the master tip: keep as-is av="$(jq -r '.actual_version // empty' "${f}")"; [ -z "${av}" ] && continue if [ -z "${best[${av}]:-}" ] || [ "${v}" -gt "${best[${av}]}" ]; then best[${av}]="${v}"; fi done for f in results/*.json; do v="$(basename "${f}" .json)" - case "${v}" in [0-9]*.[0-9]*) continue ;; esac + case "${v}" in [0-9]*.[0-9]*|master) continue ;; esac av="$(jq -r '.actual_version // empty' "${f}")"; [ -z "${av}" ] && continue if [ "${v}" = "${best[${av}]}" ]; then jq -cS --arg av "${av}" '.version = $av' "${f}" > results/.rename.tmp diff --git a/versions/list-versions.sh b/versions/list-versions.sh index 18b0623bb3..8f17b2a9ef 100755 --- a/versions/list-versions.sh +++ b/versions/list-versions.sh @@ -116,6 +116,11 @@ emit() { # version date -> resolve provider and print the line } { + # The development build: no Docker image -- the "local" provider installs it on the host + # with curl https://clickhouse.com/ | sh (see run-version.sh). Undated (the tip), it + # sorts by its server-reported version once benchmarked. + printf 'master\tlocal\t\n' + # All 1.1.x are kept (including the handful with no image, so they are at # least listed/"found"). grep -E $'^1\\.1\\.' <<<"${NORM}" | while IFS=$'\t' read -r v date; do diff --git a/versions/queries/tpcds.sql b/versions/queries/tpcds.sql index e55ac08bcd..75bdfa3482 100644 --- a/versions/queries/tpcds.sql +++ b/versions/queries/tpcds.sql @@ -50,7 +50,7 @@ SELECT ca_zip, ca_city, sum(ws_sales_price) FROM web_sales, customer, customer_a SELECT c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number, amt, profit FROM ( SELECT ss_ticket_number, ss_customer_sk, ca_city AS bought_city, sum(ss_coupon_amt) AS amt, sum(ss_net_profit) AS profit FROM store_sales, date_dim, store, household_demographics, customer_address WHERE (store_sales.ss_sold_date_sk = date_dim.d_date_sk) AND (store_sales.ss_store_sk = store.s_store_sk) AND (store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk) AND (store_sales.ss_addr_sk = customer_address.ca_address_sk) AND ((household_demographics.hd_dep_count = 4) OR (household_demographics.hd_vehicle_count = 3)) AND (date_dim.d_dow IN (6, 0)) AND (date_dim.d_year IN (1999, 1999 + 1, 1999 + 2)) AND (store.s_city IN ('Fairview', 'Midway', 'Fairview', 'Fairview', 'Fairview')) GROUP BY ss_ticket_number, ss_customer_sk, ss_addr_sk, ca_city ) AS dn, customer, customer_address AS current_addr WHERE (ss_customer_sk = c_customer_sk) AND (customer.c_current_addr_sk = current_addr.ca_address_sk) AND (current_addr.ca_city <> bought_city) ORDER BY c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number LIMIT 100 WITH v1 AS ( SELECT i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) AS sum_sales, avg(sum(ss_sales_price)) OVER (PARTITION BY i_category, i_brand, s_store_name, s_company_name, d_year) AS avg_monthly_sales, rank() OVER (PARTITION BY i_category, i_brand, s_store_name, s_company_name ORDER BY d_year, d_moy) AS rn FROM item, store_sales, date_dim, store WHERE (ss_item_sk = i_item_sk) AND (ss_sold_date_sk = d_date_sk) AND (ss_store_sk = s_store_sk) AND ( (d_year = 1999) OR ((d_year = 1999 - 1) AND (d_moy = 12)) OR ((d_year = 1999 + 1) AND (d_moy = 1)) ) GROUP BY i_category, i_brand, s_store_name, s_company_name, d_year, d_moy ), v2 AS ( SELECT v1.i_category, v1.i_brand, v1.s_store_name, v1.s_company_name, v1.d_year, v1.d_moy, v1.avg_monthly_sales, v1.sum_sales, v1_lag.sum_sales AS psum, v1_lead.sum_sales AS nsum FROM v1, v1 AS v1_lag, v1 AS v1_lead WHERE (v1.i_category = v1_lag.i_category) AND (v1.i_category = v1_lead.i_category) AND (v1.i_brand = v1_lag.i_brand) AND (v1.i_brand = v1_lead.i_brand) AND (v1.s_store_name = v1_lag.s_store_name) AND (v1.s_store_name = v1_lead.s_store_name) AND (v1.s_company_name = v1_lag.s_company_name) AND (v1.s_company_name = v1_lead.s_company_name) AND (v1.rn = v1_lag.rn + 1) AND (v1.rn = v1_lead.rn - 1) ) SELECT * FROM v2 WHERE (v1.d_year = 1999) AND (v1.avg_monthly_sales > 0) AND (CASE WHEN v1.avg_monthly_sales > 0 THEN abs(v1.sum_sales - v1.avg_monthly_sales) / v1.avg_monthly_sales ELSE NULL END > 0.1) ORDER BY v1.sum_sales - v1.avg_monthly_sales, v1.s_store_name LIMIT 100 SELECT sum(ss_quantity) FROM store_sales, store, customer_demographics, customer_address, date_dim WHERE (s_store_sk = ss_store_sk) AND (ss_sold_date_sk = d_date_sk) AND (d_year = 2000) AND ( ( (cd_demo_sk = ss_cdemo_sk) AND (cd_marital_status = 'M') AND (cd_education_status = '4 yr Degree') AND (ss_sales_price BETWEEN 100.00 AND 150.00) ) OR ( (cd_demo_sk = ss_cdemo_sk) AND (cd_marital_status = 'D') AND (cd_education_status = '2 yr Degree') AND (ss_sales_price BETWEEN 50.00 AND 100.00) ) OR ( (cd_demo_sk = ss_cdemo_sk) AND (cd_marital_status = 'S') AND (cd_education_status = 'College') AND (ss_sales_price BETWEEN 150.00 AND 200.00) ) ) AND ( ( (ss_addr_sk = ca_address_sk) AND (ca_country = 'United States') AND (ca_state IN ('CO', 'OH', 'TX')) AND (ss_net_profit BETWEEN 0 AND 2000) ) OR ( (ss_addr_sk = ca_address_sk) AND (ca_country = 'United States') AND (ca_state IN ('OR', 'MN', 'KY')) AND (ss_net_profit BETWEEN 150 AND 3000) ) OR ( (ss_addr_sk = ca_address_sk) AND (ca_country = 'United States') AND (ca_state IN ('VA', 'CA', 'MS')) AND (ss_net_profit BETWEEN 50 AND 25000) ) ) -SELECT channel, item, return_ratio, return_rank, currency_rank FROM ( SELECT 'web' AS channel, web.item, web.return_ratio, web.return_rank, web.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT ws.ws_item_sk AS item, (CAST(sum(coalesce(wr.wr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(ws.ws_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(wr.wr_return_amt, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(ws.ws_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM web_sales AS ws LEFT OUTER JOIN web_returns AS wr ON (ws.ws_order_number = wr.wr_order_number) AND (ws.ws_item_sk = wr.wr_item_sk), date_dim WHERE (wr.wr_return_amt > 10000) AND (ws.ws_net_profit > 1) AND (ws.ws_net_paid > 0) AND (ws.ws_quantity > 0) AND (ws_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY ws.ws_item_sk ) AS in_web ) AS web WHERE (web.return_rank <= 10) OR (web.currency_rank <= 10) UNION SELECT 'catalog' AS channel, catalog.item, catalog.return_ratio, catalog.return_rank, catalog.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT cs.cs_item_sk AS item, (CAST(sum(coalesce(cr.cr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(cs.cs_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(cr.cr_return_amount, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(cs.cs_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM catalog_sales AS cs LEFT OUTER JOIN catalog_returns AS cr ON (cs.cs_order_number = cr.cr_order_number) AND (cs.cs_item_sk = cr.cr_item_sk), date_dim WHERE (cr.cr_return_amount > 10000) AND (cs.cs_net_profit > 1) AND (cs.cs_net_paid > 0) AND (cs.cs_quantity > 0) AND (cs_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY cs.cs_item_sk ) AS in_cat ) AS catalog WHERE (catalog.return_rank <= 10) OR (catalog.currency_rank <= 10) UNION SELECT 'store' AS channel, store.item, store.return_ratio, store.return_rank, store.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT sts.ss_item_sk AS item, (CAST(sum(coalesce(sr.sr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(sts.ss_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(sr.sr_return_amt, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM store_sales AS sts LEFT OUTER JOIN store_returns AS sr ON (sts.ss_ticket_number = sr.sr_ticket_number) AND (sts.ss_item_sk = sr.sr_item_sk), date_dim WHERE (sr.sr_return_amt > 10000) AND (sts.ss_net_profit > 1) AND (sts.ss_net_paid > 0) AND (sts.ss_quantity > 0) AND (ss_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY sts.ss_item_sk ) AS in_store ) AS store WHERE (store.return_rank <= 10) OR (store.currency_rank <= 10) ) ORDER BY 1, 4, 5, 2 LIMIT 100 +SELECT channel, item, return_ratio, return_rank, currency_rank FROM ( SELECT 'web' AS channel, web.item, web.return_ratio, web.return_rank, web.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT ws.ws_item_sk AS item, (CAST(sum(coalesce(wr.wr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(ws.ws_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(wr.wr_return_amt, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(ws.ws_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM web_sales AS ws LEFT OUTER JOIN web_returns AS wr ON (ws.ws_order_number = wr.wr_order_number) AND (ws.ws_item_sk = wr.wr_item_sk), date_dim WHERE (wr.wr_return_amt > 10000) AND (ws.ws_net_profit > 1) AND (ws.ws_net_paid > 0) AND (ws.ws_quantity > 0) AND (ws_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY ws.ws_item_sk ) AS in_web ) AS web WHERE (web.return_rank <= 10) OR (web.currency_rank <= 10) UNION DISTINCT SELECT 'catalog' AS channel, catalog.item, catalog.return_ratio, catalog.return_rank, catalog.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT cs.cs_item_sk AS item, (CAST(sum(coalesce(cr.cr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(cs.cs_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(cr.cr_return_amount, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(cs.cs_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM catalog_sales AS cs LEFT OUTER JOIN catalog_returns AS cr ON (cs.cs_order_number = cr.cr_order_number) AND (cs.cs_item_sk = cr.cr_item_sk), date_dim WHERE (cr.cr_return_amount > 10000) AND (cs.cs_net_profit > 1) AND (cs.cs_net_paid > 0) AND (cs.cs_quantity > 0) AND (cs_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY cs.cs_item_sk ) AS in_cat ) AS catalog WHERE (catalog.return_rank <= 10) OR (catalog.currency_rank <= 10) UNION DISTINCT SELECT 'store' AS channel, store.item, store.return_ratio, store.return_rank, store.currency_rank FROM ( SELECT item, return_ratio, currency_ratio, rank() OVER (ORDER BY return_ratio) AS return_rank, rank() OVER (ORDER BY currency_ratio) AS currency_rank FROM ( SELECT sts.ss_item_sk AS item, (CAST(sum(coalesce(sr.sr_return_quantity, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(sts.ss_quantity, 0)) AS decimal(15, 4))) AS return_ratio, (CAST(sum(coalesce(sr.sr_return_amt, 0)) AS decimal(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid, 0)) AS decimal(15, 4))) AS currency_ratio FROM store_sales AS sts LEFT OUTER JOIN store_returns AS sr ON (sts.ss_ticket_number = sr.sr_ticket_number) AND (sts.ss_item_sk = sr.sr_item_sk), date_dim WHERE (sr.sr_return_amt > 10000) AND (sts.ss_net_profit > 1) AND (sts.ss_net_paid > 0) AND (sts.ss_quantity > 0) AND (ss_sold_date_sk = d_date_sk) AND (d_year = 2001) AND (d_moy = 12) GROUP BY sts.ss_item_sk ) AS in_store ) AS store WHERE (store.return_rank <= 10) OR (store.currency_rank <= 10) ) ORDER BY 1, 4, 5, 2 LIMIT 100 SELECT s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip, sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk <= 30) THEN 1 ELSE 0 END) AS "30 days", sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk > 30) AND (sr_returned_date_sk - ss_sold_date_sk <= 60) THEN 1 ELSE 0 END) AS "31-60 days", sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk > 60) AND (sr_returned_date_sk - ss_sold_date_sk <= 90) THEN 1 ELSE 0 END) AS "61-90 days", sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk > 90) AND (sr_returned_date_sk - ss_sold_date_sk <= 120) THEN 1 ELSE 0 END) AS "91-120 days", sum(CASE WHEN (sr_returned_date_sk - ss_sold_date_sk > 120) THEN 1 ELSE 0 END) AS ">120 days" FROM store_sales, store_returns, store, date_dim AS d1, date_dim AS d2 WHERE (d2.d_year = 2001) AND (d2.d_moy = 8) AND (ss_ticket_number = sr_ticket_number) AND (ss_item_sk = sr_item_sk) AND (ss_sold_date_sk = d1.d_date_sk) AND (sr_returned_date_sk = d2.d_date_sk) AND (ss_customer_sk = sr_customer_sk) AND (ss_store_sk = s_store_sk) GROUP BY s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip ORDER BY s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip LIMIT 100 WITH web_v1 AS ( SELECT ws_item_sk AS item_sk, d_date, sum(sum(ws_sales_price)) OVER (PARTITION BY ws_item_sk ORDER BY d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cume_sales FROM web_sales, date_dim WHERE (ws_sold_date_sk = d_date_sk) AND (d_month_seq BETWEEN 1200 AND 1200 + 11) AND (ws_item_sk IS NOT NULL) GROUP BY ws_item_sk, d_date ), store_v1 AS ( SELECT ss_item_sk AS item_sk, d_date, sum(sum(ss_sales_price)) OVER (PARTITION BY ss_item_sk ORDER BY d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cume_sales FROM store_sales, date_dim WHERE (ss_sold_date_sk = d_date_sk) AND (d_month_seq BETWEEN 1200 AND 1200 + 11) AND (ss_item_sk IS NOT NULL) GROUP BY ss_item_sk, d_date ) SELECT * FROM ( SELECT item_sk, d_date, web_sales, store_sales, max(web_sales) OVER (PARTITION BY item_sk ORDER BY d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS web_cumulative, max(store_sales) OVER (PARTITION BY item_sk ORDER BY d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS store_cumulative FROM ( SELECT CASE WHEN web.item_sk IS NOT NULL THEN web.item_sk ELSE store.item_sk END AS item_sk, CASE WHEN web.d_date IS NOT NULL THEN web.d_date ELSE store.d_date END AS d_date, web.cume_sales AS web_sales, store.cume_sales AS store_sales FROM web_v1 AS web FULL OUTER JOIN store_v1 AS store ON (web.item_sk = store.item_sk) AND (web.d_date = store.d_date) ) AS x ) AS y WHERE (web_cumulative > store_cumulative) ORDER BY item_sk, d_date LIMIT 100 SELECT dt.d_year, item.i_brand_id AS brand_id, item.i_brand AS brand, sum(ss_ext_sales_price) AS ext_price FROM date_dim AS dt, store_sales, item WHERE (dt.d_date_sk = store_sales.ss_sold_date_sk) AND (store_sales.ss_item_sk = item.i_item_sk) AND (item.i_manager_id = 1) AND (dt.d_moy = 11) AND (dt.d_year = 2000) GROUP BY dt.d_year, item.i_brand, item.i_brand_id ORDER BY dt.d_year, ext_price DESC, brand_id LIMIT 100 @@ -76,7 +76,7 @@ SELECT i_brand_id AS brand_id, i_brand AS brand, t_hour, t_minute, sum(ext_price SELECT i_item_desc, w_warehouse_name, d1.d_week_seq, sum(multiIf(p_promo_sk IS NULL, 1, 0)) AS no_promo, sum(multiIf(p_promo_sk IS NOT NULL, 1, 0)) AS promo, count(*) AS total_cnt FROM catalog_sales INNER JOIN inventory ON cs_item_sk = inv_item_sk INNER JOIN warehouse ON w_warehouse_sk = inv_warehouse_sk INNER JOIN item ON i_item_sk = cs_item_sk INNER JOIN customer_demographics ON cs_bill_cdemo_sk = cd_demo_sk INNER JOIN household_demographics ON cs_bill_hdemo_sk = hd_demo_sk INNER JOIN date_dim AS d1 ON cs_sold_date_sk = d1.d_date_sk INNER JOIN date_dim AS d2 ON inv_date_sk = d2.d_date_sk INNER JOIN date_dim AS d3 ON cs_ship_date_sk = d3.d_date_sk LEFT JOIN promotion ON cs_promo_sk = p_promo_sk LEFT JOIN catalog_returns ON (cr_item_sk = cs_item_sk) AND (cr_order_number = cs_order_number) WHERE (d1.d_week_seq = d2.d_week_seq) AND (inv_quantity_on_hand < cs_quantity) AND (d3.d_date > (d1.d_date + 5)) AND (hd_buy_potential = '>10000') AND (d1.d_year = 1999) AND (cd_marital_status = 'D') GROUP BY i_item_desc, w_warehouse_name, d_week_seq ORDER BY total_cnt DESC, i_item_desc, w_warehouse_name, d_week_seq LIMIT 100 SELECT c_last_name, c_first_name, c_salutation, c_preferred_cust_flag, ss_ticket_number, cnt FROM ( SELECT ss_ticket_number, ss_customer_sk, count(*) AS cnt FROM store_sales, date_dim, store, household_demographics WHERE (store_sales.ss_sold_date_sk = date_dim.d_date_sk) AND (store_sales.ss_store_sk = store.s_store_sk) AND (store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk) AND ((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 2)) AND ((household_demographics.hd_buy_potential = '>10000') OR (household_demographics.hd_buy_potential = 'Unknown')) AND (household_demographics.hd_vehicle_count > 0) AND (multiIf(household_demographics.hd_vehicle_count > 0, household_demographics.hd_dep_count / household_demographics.hd_vehicle_count, NULL) > 1) AND (date_dim.d_year IN (1999, 1999 + 1, 1999 + 2)) AND (store.s_county IN ('Williamson County', 'Franklin Parish', 'Bronx County', 'Orange County')) GROUP BY ss_ticket_number, ss_customer_sk ) AS dj, customer WHERE (ss_customer_sk = c_customer_sk) AND ((cnt >= 1) AND (cnt <= 5)) ORDER BY cnt DESC, c_last_name WITH year_total AS ( SELECT c_customer_id AS customer_id, c_first_name AS customer_first_name, c_last_name AS customer_last_name, d_year AS year, sum(ss_net_paid) AS year_total, 's' AS sale_type FROM customer, store_sales, date_dim WHERE (c_customer_sk = ss_customer_sk) AND (ss_sold_date_sk = d_date_sk) AND (d_year IN (2001, 2001 + 1)) GROUP BY c_customer_id, c_first_name, c_last_name, d_year UNION ALL SELECT c_customer_id AS customer_id, c_first_name AS customer_first_name, c_last_name AS customer_last_name, d_year AS year, sum(ws_net_paid) AS year_total, 'w' AS sale_type FROM customer, web_sales, date_dim WHERE (c_customer_sk = ws_bill_customer_sk) AND (ws_sold_date_sk = d_date_sk) AND (d_year IN (2001, 2001 + 1)) GROUP BY c_customer_id, c_first_name, c_last_name, d_year ) SELECT t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name FROM year_total AS t_s_firstyear, year_total AS t_s_secyear, year_total AS t_w_firstyear, year_total AS t_w_secyear WHERE (t_s_secyear.customer_id = t_s_firstyear.customer_id) AND (t_s_firstyear.customer_id = t_w_secyear.customer_id) AND (t_s_firstyear.customer_id = t_w_firstyear.customer_id) AND (t_s_firstyear.sale_type = 's') AND (t_w_firstyear.sale_type = 'w') AND (t_s_secyear.sale_type = 's') AND (t_w_secyear.sale_type = 'w') AND (t_s_firstyear.year = 2001) AND (t_s_secyear.year = (2001 + 1)) AND (t_w_firstyear.year = 2001) AND (t_w_secyear.year = (2001 + 1)) AND (t_s_firstyear.year_total > 0) AND (t_w_firstyear.year_total > 0) AND (multiIf(t_w_firstyear.year_total > 0, t_w_secyear.year_total / t_w_firstyear.year_total, NULL) > multiIf(t_s_firstyear.year_total > 0, t_s_secyear.year_total / t_s_firstyear.year_total, NULL)) ORDER BY 1, 2, 3 LIMIT 100 -WITH all_sales AS ( SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, SUM(sales_cnt) AS sales_cnt, SUM(sales_amt) AS sales_amt FROM ( SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, cs_quantity - COALESCE(cr_return_quantity, 0) AS sales_cnt, cs_ext_sales_price - COALESCE(cr_return_amount, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM catalog_sales INNER JOIN item ON i_item_sk = cs_item_sk INNER JOIN date_dim ON d_date_sk = cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number = cr_order_number) AND (cs_item_sk = cr_item_sk) WHERE (i_category = 'Books') UNION SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, ss_quantity - COALESCE(sr_return_quantity, 0) AS sales_cnt, ss_ext_sales_price - COALESCE(sr_return_amt, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM store_sales INNER JOIN item ON i_item_sk = ss_item_sk INNER JOIN date_dim ON d_date_sk = ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number = sr_ticket_number) AND (ss_item_sk = sr_item_sk) WHERE (i_category = 'Books') UNION SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, ws_quantity - COALESCE(wr_return_quantity, 0) AS sales_cnt, ws_ext_sales_price - COALESCE(wr_return_amt, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM web_sales INNER JOIN item ON i_item_sk = ws_item_sk INNER JOIN date_dim ON d_date_sk = ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number = wr_order_number) AND (ws_item_sk = wr_item_sk) WHERE (i_category = 'Books') ) AS sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id ) SELECT prev_yr.d_year AS prev_year, curr_yr.d_year AS year, curr_yr.i_brand_id, curr_yr.i_class_id, curr_yr.i_category_id, curr_yr.i_manufact_id, prev_yr.sales_cnt AS prev_yr_cnt, curr_yr.sales_cnt AS curr_yr_cnt, curr_yr.sales_cnt - prev_yr.sales_cnt AS sales_cnt_diff, curr_yr.sales_amt - prev_yr.sales_amt AS sales_amt_diff FROM all_sales AS curr_yr, all_sales AS prev_yr WHERE (curr_yr.i_brand_id = prev_yr.i_brand_id) AND (curr_yr.i_class_id = prev_yr.i_class_id) AND (curr_yr.i_category_id = prev_yr.i_category_id) AND (curr_yr.i_manufact_id = prev_yr.i_manufact_id) AND (curr_yr.d_year = 2002) AND (prev_yr.d_year = (2002 - 1)) AND ((CAST(curr_yr.sales_cnt, 'DECIMAL(17, 2)') / CAST(prev_yr.sales_cnt, 'DECIMAL(17, 2)')) < 0.9) ORDER BY sales_cnt_diff, sales_amt_diff LIMIT 100 +WITH all_sales AS ( SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, SUM(sales_cnt) AS sales_cnt, SUM(sales_amt) AS sales_amt FROM ( SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, cs_quantity - COALESCE(cr_return_quantity, 0) AS sales_cnt, cs_ext_sales_price - COALESCE(cr_return_amount, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM catalog_sales INNER JOIN item ON i_item_sk = cs_item_sk INNER JOIN date_dim ON d_date_sk = cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number = cr_order_number) AND (cs_item_sk = cr_item_sk) WHERE (i_category = 'Books') UNION DISTINCT SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, ss_quantity - COALESCE(sr_return_quantity, 0) AS sales_cnt, ss_ext_sales_price - COALESCE(sr_return_amt, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM store_sales INNER JOIN item ON i_item_sk = ss_item_sk INNER JOIN date_dim ON d_date_sk = ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number = sr_ticket_number) AND (ss_item_sk = sr_item_sk) WHERE (i_category = 'Books') UNION DISTINCT SELECT d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, ws_quantity - COALESCE(wr_return_quantity, 0) AS sales_cnt, ws_ext_sales_price - COALESCE(wr_return_amt, CAST('0.0', 'Decimal(7, 2)')) AS sales_amt FROM web_sales INNER JOIN item ON i_item_sk = ws_item_sk INNER JOIN date_dim ON d_date_sk = ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number = wr_order_number) AND (ws_item_sk = wr_item_sk) WHERE (i_category = 'Books') ) AS sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id ) SELECT prev_yr.d_year AS prev_year, curr_yr.d_year AS year, curr_yr.i_brand_id, curr_yr.i_class_id, curr_yr.i_category_id, curr_yr.i_manufact_id, prev_yr.sales_cnt AS prev_yr_cnt, curr_yr.sales_cnt AS curr_yr_cnt, curr_yr.sales_cnt - prev_yr.sales_cnt AS sales_cnt_diff, curr_yr.sales_amt - prev_yr.sales_amt AS sales_amt_diff FROM all_sales AS curr_yr, all_sales AS prev_yr WHERE (curr_yr.i_brand_id = prev_yr.i_brand_id) AND (curr_yr.i_class_id = prev_yr.i_class_id) AND (curr_yr.i_category_id = prev_yr.i_category_id) AND (curr_yr.i_manufact_id = prev_yr.i_manufact_id) AND (curr_yr.d_year = 2002) AND (prev_yr.d_year = (2002 - 1)) AND ((CAST(curr_yr.sales_cnt, 'DECIMAL(17, 2)') / CAST(prev_yr.sales_cnt, 'DECIMAL(17, 2)')) < 0.9) ORDER BY sales_cnt_diff, sales_amt_diff LIMIT 100 SELECT channel, col_name, d_year, d_qoy, i_category, COUNT(*) AS sales_cnt, SUM(ext_sales_price) AS sales_amt FROM ( SELECT 'store' AS channel, 'ss_store_sk' AS col_name, d_year, d_qoy, i_category, ss_ext_sales_price AS ext_sales_price FROM store_sales, item, date_dim WHERE (ss_store_sk IS NULL) AND (ss_sold_date_sk = d_date_sk) AND (ss_item_sk = i_item_sk) UNION ALL SELECT 'web' AS channel, 'ws_ship_customer_sk' AS col_name, d_year, d_qoy, i_category, ws_ext_sales_price AS ext_sales_price FROM web_sales, item, date_dim WHERE (ws_ship_customer_sk IS NULL) AND (ws_sold_date_sk = d_date_sk) AND (ws_item_sk = i_item_sk) UNION ALL SELECT 'catalog' AS channel, 'cs_ship_addr_sk' AS col_name, d_year, d_qoy, i_category, cs_ext_sales_price AS ext_sales_price FROM catalog_sales, item, date_dim WHERE (cs_ship_addr_sk IS NULL) AND (cs_sold_date_sk = d_date_sk) AND (cs_item_sk = i_item_sk) ) AS foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category LIMIT 100 WITH ss AS ( SELECT s_store_sk, sum(ss_ext_sales_price) AS sales, sum(ss_net_profit) AS profit FROM store_sales, date_dim, store WHERE (ss_sold_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) AND (ss_store_sk = s_store_sk) GROUP BY s_store_sk ), sr AS ( SELECT s_store_sk, sum(sr_return_amt) AS returns, sum(sr_net_loss) AS profit_loss FROM store_returns, date_dim, store WHERE (sr_returned_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) AND (sr_store_sk = s_store_sk) GROUP BY s_store_sk ), cs AS ( SELECT cs_call_center_sk, sum(cs_ext_sales_price) AS sales, sum(cs_net_profit) AS profit FROM catalog_sales, date_dim WHERE (cs_sold_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) GROUP BY cs_call_center_sk ), cr AS ( SELECT cr_call_center_sk, sum(cr_return_amount) AS returns, sum(cr_net_loss) AS profit_loss FROM catalog_returns, date_dim WHERE (cr_returned_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) GROUP BY cr_call_center_sk ), ws AS ( SELECT wp_web_page_sk, sum(ws_ext_sales_price) AS sales, sum(ws_net_profit) AS profit FROM web_sales, date_dim, web_page WHERE (ws_sold_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) AND (ws_web_page_sk = wp_web_page_sk) GROUP BY wp_web_page_sk ), wr AS ( SELECT wp_web_page_sk, sum(wr_return_amt) AS returns, sum(wr_net_loss) AS profit_loss FROM web_returns, date_dim, web_page WHERE (wr_returned_date_sk = d_date_sk) AND ((d_date >= CAST('2000-08-23', 'date')) AND (d_date <= (CAST('2000-08-23', 'date') + INTERVAL 30 DAY))) AND (wr_web_page_sk = wp_web_page_sk) GROUP BY wp_web_page_sk ) SELECT channel, id, sum(sales) AS sales, sum(returns) AS returns, sum(profit) AS profit FROM ( SELECT 'store channel' AS channel, ss.s_store_sk AS id, sales, coalesce(returns, 0) AS returns, profit - coalesce(profit_loss, 0) AS profit FROM ss LEFT JOIN sr ON ss.s_store_sk = sr.s_store_sk UNION ALL SELECT 'catalog channel' AS channel, cs_call_center_sk AS id, sales, returns, profit - profit_loss AS profit FROM cs, cr UNION ALL SELECT 'web channel' AS channel, ws.wp_web_page_sk AS id, sales, coalesce(returns, 0) AS returns, profit - coalesce(profit_loss, 0) AS profit FROM ws LEFT JOIN wr ON ws.wp_web_page_sk = wr.wp_web_page_sk ) AS x GROUP BY channel, id WITH ROLLUP ORDER BY channel, id LIMIT 100 WITH ws AS ( SELECT d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk AS ws_customer_sk, sum(ws_quantity) AS ws_qty, sum(ws_wholesale_cost) AS ws_wc, sum(ws_sales_price) AS ws_sp FROM web_sales LEFT JOIN web_returns ON (wr_order_number = ws_order_number) AND (ws_item_sk = wr_item_sk) INNER JOIN date_dim ON ws_sold_date_sk = d_date_sk WHERE wr_order_number IS NULL GROUP BY d_year, ws_item_sk, ws_bill_customer_sk ), cs AS ( SELECT d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk AS cs_customer_sk, sum(cs_quantity) AS cs_qty, sum(cs_wholesale_cost) AS cs_wc, sum(cs_sales_price) AS cs_sp FROM catalog_sales LEFT JOIN catalog_returns ON (cr_order_number = cs_order_number) AND (cs_item_sk = cr_item_sk) INNER JOIN date_dim ON cs_sold_date_sk = d_date_sk WHERE cr_order_number IS NULL GROUP BY d_year, cs_item_sk, cs_bill_customer_sk ), ss AS ( SELECT d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) AS ss_qty, sum(ss_wholesale_cost) AS ss_wc, sum(ss_sales_price) AS ss_sp FROM store_sales LEFT JOIN store_returns ON (sr_ticket_number = ss_ticket_number) AND (ss_item_sk = sr_item_sk) INNER JOIN date_dim ON ss_sold_date_sk = d_date_sk WHERE sr_ticket_number IS NULL GROUP BY d_year, ss_item_sk, ss_customer_sk ) SELECT ss_sold_year, ss_item_sk, ss_customer_sk, round(ss_qty / (coalesce(ws_qty, 0) + coalesce(cs_qty, 0)), 2) AS ratio, ss_qty AS store_qty, ss_wc AS store_wholesale_cost, ss_sp AS store_sales_price, coalesce(ws_qty, 0) + coalesce(cs_qty, 0) AS other_chan_qty, coalesce(ws_wc, 0) + coalesce(cs_wc, 0) AS other_chan_wholesale_cost, coalesce(ws_sp, 0) + coalesce(cs_sp, 0) AS other_chan_sales_price FROM ss LEFT JOIN ws ON (ws_sold_year = ss_sold_year) AND (ws_item_sk = ss_item_sk) AND (ws_customer_sk = ss_customer_sk) LEFT JOIN cs ON (cs_sold_year = ss_sold_year) AND (cs_item_sk = ss_item_sk) AND (cs_customer_sk = ss_customer_sk) WHERE ((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)) AND (ss_sold_year = 2000) ORDER BY ss_sold_year, ss_item_sk, ss_customer_sk, ss_qty DESC, ss_wc DESC, ss_sp DESC, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio LIMIT 100 diff --git a/versions/run-version.sh b/versions/run-version.sh index e4c126d98d..3b913c8d0c 100755 --- a/versions/run-version.sh +++ b/versions/run-version.sh @@ -38,6 +38,14 @@ PHASE="${3:-all}" [ -z "${IMAGE}" ] && IMAGE="$(./list-versions.sh | awk -v v="${VERSION}" '$1==v{print $2}')" [ -z "${IMAGE}" ] && { echo "no image for ${VERSION}" >&2; exit 1; } +# "local" provider (used for the master/dev build): no Docker -- install ClickHouse on the +# host with the official one-line installer (curl https://clickhouse.com/ | sh) and run the +# server as a background process out of LOCAL_DIR (which then holds its data). Loading, +# queries and sizing are otherwise identical to the Docker path. +LOCAL=""; [ "${IMAGE}" = "local" ] && LOCAL=1 +LOCAL_DIR="${HERE}/.local-server" +LOCAL_BIN="${LOCAL_DIR}/clickhouse" + CONTAINER="chver_${VERSION//[^0-9A-Za-z]/_}" OUT="${HERE}/results/${VERSION}.json" # Per-table load timings, written during the load phase and read by the bench @@ -69,7 +77,14 @@ declare -A TABLES=( # late crash can't take down the earlier datasets' results as collateral. QUERY_ORDER="mgbench ssb hits uk ontime taxi coffeeshop tpch tpcds job" -cleanup() { sudo docker rm -f "${CONTAINER}" >/dev/null 2>&1; } +cleanup() { + if [ -n "${LOCAL}" ]; then + [ -f "${LOCAL_DIR}/server.pid" ] && kill "$(cat "${LOCAL_DIR}/server.pid")" 2>/dev/null + pkill -f "${LOCAL_BIN} server" 2>/dev/null + return 0 + fi + sudo docker rm -f "${CONTAINER}" >/dev/null 2>&1 +} # The load phase must leave the container running for the later bench phase; # all/bench tear it down on exit. [ "${PHASE}" != "load" ] && trap cleanup EXIT @@ -100,8 +115,11 @@ exec_client() { sudo ${CH_TIMEOUT:+timeout ${CH_TIMEOUT}} docker exec -i -e H sidecar_client() { sudo ${CH_TIMEOUT:+timeout ${CH_TIMEOUT}} docker run --rm -i -e HOME=/tmp -e TZ=UTC \ -v /usr/share/zoneinfo:/usr/share/zoneinfo:ro \ --network "container:${CONTAINER}" "${CLIENT_IMAGE}" "$@"; } +# local provider: the installed binary, connecting to the host server over TCP. +local_client() { ${CH_TIMEOUT:+timeout ${CH_TIMEOUT}} env HOME=/tmp TZ=UTC "${LOCAL_BIN}" client "$@"; } client() { case "${CLIENT_MODE}" in + local) local_client "$@" ;; sidecar) sidecar_client "$@" ;; *) exec_client "$@" ;; esac @@ -161,6 +179,24 @@ ensure_built_image() { start_server() { cleanup + if [ -n "${LOCAL}" ]; then + # No Docker: install with the official one-liner (once) and run the server on the + # host, out of LOCAL_DIR so its data lands there. + echo "starting ${VERSION} via the local installer (curl https://clickhouse.com/ | sh)" >&2 + mkdir -p "${LOCAL_DIR}" + [ -x "${LOCAL_BIN}" ] || ( cd "${LOCAL_DIR}" && curl -fsSL https://clickhouse.com/ | sh ) >&2 + [ -x "${LOCAL_BIN}" ] || { echo "local install failed: ${LOCAL_BIN} missing" >&2; return 1; } + ( cd "${LOCAL_DIR}" && HOME=/tmp TZ=UTC nohup ./clickhouse server > server.log 2>&1 & echo $! > server.pid ) + CLIENT_MODE=local + local i + for i in $(seq 1 "${READY_TIMEOUT:-90}"); do + local_client --query "SELECT 1" >/dev/null 2>&1 && return 0 + sleep 1 + done + echo "local server ${VERSION} did not become ready; last log lines:" >&2 + tail -20 "${LOCAL_DIR}/server.log" >&2 2>&1 || true + return 1 + fi if [ "${IMAGE}" = "package" ]; then # Fallback provider: install the .deb release into a stock Ubuntu image. echo "starting ${VERSION} via package-in-ubuntu fallback" >&2 @@ -368,6 +404,13 @@ launch_daemon_in_container() { } wait_alive() { local i; for i in $(seq 1 "${1:-180}"); do server_alive && return 0; sleep 1; done; return 1; } revive_server() { + if [ -n "${LOCAL}" ]; then + echo "${VERSION}: relaunching local server; recent log:" >&2 + tail -8 "${LOCAL_DIR}/server.log" 2>&1 | sed 's/^/ | /' >&2 || true + ( cd "${LOCAL_DIR}" && HOME=/tmp TZ=UTC nohup ./clickhouse server >> server.log 2>&1 & echo $! > server.pid ) + wait_alive "${REVIVE_TIMEOUT:-180}" && { echo "${VERSION}: local server back up" >&2; return 0; } + echo "${VERSION}: local server did not come back" >&2; return 1 + fi local running running="$(sudo docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null)" echo "${VERSION}: reviving server (container running=${running:-unknown}); recent container logs:" >&2 @@ -403,7 +446,7 @@ report_sizes() { # The benchmark tables live in per-dataset databases; exclude the server's # own system databases. out=$(client --query "SELECT database, table, sum(bytes_on_disk) AS size FROM system.parts WHERE database NOT IN ('system', 'information_schema', 'INFORMATION_SCHEMA') GROUP BY database, table ORDER BY database, table FORMAT TabSeparated" /dev/null) - if [ -n "${out}" ]; then + if [ -n "${out}" ] || [ -n "${LOCAL}" ]; then printf '%s\n' "${out}" else echo "(system.parts unavailable — measuring the data directory)" @@ -424,6 +467,7 @@ fmt_err() { printf '%s' "$1" | tr '\n\t' ' ' | sed 's/ */ /g; s/^ //' | cut -c # so 1.1.54378 and the bare 53982 both sort *below* any calendar release. version_key() { local v="$1" a b c d + [ "$v" = "master" ] && { echo "999999 0 0 0"; return; } # the dev build is the newest if [[ "$v" =~ ^[0-9]+$ ]]; then echo "1 1 $v 0"; return; fi IFS='.' read -r a b c d _ <<<"$v" echo "${a:-0} ${b:-0} ${c:-0} ${d:-0}" @@ -516,6 +560,12 @@ run_query() { # Attach to an already-running, already-loaded container (bench phase): detect # the client mode without (re)starting or wiping the container. detect_client() { + if [ -n "${LOCAL}" ]; then + CLIENT_MODE=local + local i + for i in $(seq 1 30); do local_client --query "SELECT 1" >/dev/null 2>&1 && return 0; sleep 1; done + echo "local server for ${VERSION} not answering" >&2; return 1 + fi sudo docker ps -q -f "name=^${CONTAINER}$" | grep -q . || { echo "container ${CONTAINER} not running" >&2; return 1; } [ "${IMAGE}" != "package" ] && [ -n "${CLIENT_IMAGE}" ] && sudo docker pull "${CLIENT_IMAGE}" >/dev/null 2>&1 local i @@ -547,7 +597,7 @@ emit_load_time_json() { emit_data_size_json() { local out loaded=" ${1:-} " out=$(client --query "SELECT database, sum(bytes_on_disk) FROM system.parts WHERE active AND database NOT IN ('system', 'information_schema', 'INFORMATION_SCHEMA') GROUP BY database FORMAT TabSeparated" /dev/null) - if [ -z "${out}" ]; then + if [ -z "${out}" ] && [ -z "${LOCAL}" ]; then # No system.parts: sum the byte size of each dataset's data directory in the # container (works for both Log and MergeTree; -L follows the store/ symlinks). out=$(sudo docker exec "${CONTAINER}" sh -c ' @@ -579,7 +629,7 @@ release_date() { server_version() { local v rev v=$(client --query "SELECT version()" 2>/dev/null | tr -d '\r') - if [ -z "${v}" ]; then + if [ -z "${v}" ] && [ -z "${LOCAL}" ]; then rev=$(sudo docker exec "${CONTAINER}" cat /clickhouse-revision 2>/dev/null | tr -d '\r\n ') [ -n "${rev}" ] && v="0.0.${rev}" fi