- Customer segmentation with density clustering
- Assign a new customer and update the segments
- Compare search algorithms on a delivery network
- Data quality gate before training
- Audit a database column with a custom rule
- Customer segments straight from a SQL table
- Linear regression solved by QR decomposition
- Is the new model really better? Mann-Whitney on CV folds
- Class imbalance audit and balanced class weights
- PCA: how much variance does the first component keep?
- Random forest with a hold-out test set
- Benchmark five classifiers with a cross validation
- Tune a gradient boosting with a grid search
- Save a trained model and score with it later
- Map the operating modes of a machine with a self-organizing map
- Clusters of different densities with HDBSCAN
- Predict the next step of a user journey
- Stacking against the single models
Customer segmentation with density clustering
Group 14 customers by annual spend and visit frequency with a DBSCAN-like clustering (radius 1.0, at least 3 neighbours), then report the size and centroid of each segment, the number of outliers and the distance between the low and high spend centroids.
#1. Customers: annual spend in kEUR, store visits per month; -> "[cust]" "[[1.2, 1.0], [1.5, 1.2], [1.1, 0.8], [1.4, 1.5], [1.3, 1.1], [4.8, 3.9], [5.1, 4.2], [4.6, 4.0], [5.3, 3.7], [9.5, 2.0], [9.9, 2.3], [10.2, 1.8], [9.7, 2.1], [7.0, 9.0]]"; #2. Density clustering: radius 1.0, at least 3 customers per segment; ml cluster load_from_json "rx_seg" 1.0 3 [cust]; -> "[k]" (ml cluster nb "rx_seg"); -> "[report]" ""; -> "[clustered]" 0; #3. Size and centroid of each segment; for (-> "[i]" 0) (< [i] [k]) (++ "[i]") { -> "[n]" (ml cluster nb_point "rx_seg" [i]); -> "[sx]" 0; -> "[sy]" 0; for (-> "[j]" 0) (< [j] [n]) (++ "[j]") { json load "rx_pt" (ml cluster point_get "rx_seg" [i] [j]); -> "[sx]" (+ [sx] (json select "rx_pt" "/[0]")); -> "[sy]" (+ [sy] (json select "rx_pt" "/[1]")); }; -> "[clustered]" (+ [clustered] [n]); -> (concat "[cx" [i] "]") (math round (/ [sx] [n]) 2); -> (concat "[cy" [i] "]") (math round (/ [sy] [n]) 2); -> "[report]" (concat [report] "segment " [i] ": " [n] " customers, centroid " (env get var (concat "[cx" [i] "]")) " kEUR / " (env get var (concat "[cy" [i] "]")) " visits; "); }; json unload "rx_pt"; #4. Distance between the low and high spend centroids, and noise; -> "[gap]" (math round (ml cluster distance "rx_seg" [cx0] [cy0] [cx2] [cy2]) 2); ml cluster close "rx_seg"; concat [k] " segments | " [report] "outliers: " (- 14 [clustered]) " | low-spend to high-spend distance: " [gap]
3 segments | segment 0: 5 customers, centroid 1.3 kEUR / 1.12 visits; segment 1: 4 customers, centroid 4.95 kEUR / 3.95 visits; segment 2: 4 customers, centroid 9.83 kEUR / 2.05 visits; outliers: 1 | low-spend to high-spend distance: 8.58
- Read a whole segment at once with
ml cluster pointsto feedstat summary. - Load the points straight from a table with
ml cluster loadand a connection id.
Assign a new customer and update the segments
Keep last month segments in memory: assign a new customer to the segment with the smallest mean distance, move a customer whose spend grew, remove a churned customer, then draw the segments.
#1. Reload last month segments if they are not in memory; if (== (ml cluster exist "rx_seg2") 0) { ml cluster load_from_json "rx_seg2" 1.0 3 "[[1.2, 1.0], [1.5, 1.2], [1.1, 0.8], [1.4, 1.5], [1.3, 1.1], [4.8, 3.9], [5.1, 4.2], [4.6, 4.0], [5.3, 3.7], [9.5, 2.0], [9.9, 2.3], [10.2, 1.8], [9.7, 2.1]]"; }; #2. New customer: nearest segment by mean distance to its members; -> "[nx]" 5.6; -> "[ny]" 3.4; -> "[best]" -1; -> "[dmin]" 1000000; for (-> "[i]" 0) (< [i] (ml cluster nb "rx_seg2")) (++ "[i]") { json load "rx_pts" (ml cluster points "rx_seg2" [i]); -> "[s]" 0; for (-> "[j]" 0) (< [j] (json count "rx_pts" /)) (++ "[j]") { -> "[s]" (+ [s] (ml cluster distance "rx_seg2" [nx] [ny] (json select "rx_pts" (concat "/[" [j] "]/[0]")) (json select "rx_pts" (concat "/[" [j] "]/[1]")))); }; -> "[d]" (/ [s] (json count "rx_pts" /)); if (< [d] [dmin]) {-> "[dmin]" [d]; -> "[best]" [i];}; }; json unload "rx_pts"; ml cluster point_add "rx_seg2" [best] [nx] [ny]; #3. A customer moved up, another one churned; ml cluster point_update "rx_seg2" 0 3 2.1 1.9; ml cluster point_delete "rx_seg2" 2 1; #4. Draw the updated segments; ml cluster xy_scatter "rx_seg2"
The scatter chart opens in a window of the editor.
- Run
ml cluster showto list the clusters of your session. - Close it with
ml cluster close;ml cluster close_allfrees every cluster of the session.
Compare search algorithms on a delivery network
Model a one-way road network with travel times in minutes and find the fastest route from the depot to a store, comparing optimal algorithms (Dijkstra, A*, Bellman-Ford, IDA*) with blind or greedy ones (BFS, DFS, hill climbing).
#1. One-way road network, travel time in minutes; ml h_node load_from_json "rx_road" true "[[\"Depot\", \"A\", 12], [\"Depot\", \"B\", 7], [\"B\", \"A\", 3], [\"A\", \"C\", 9], [\"B\", \"D\", 15], [\"C\", \"D\", 4], [\"C\", \"Store\", 11], [\"D\", \"Store\", 6]]"; #2. Search problem: leave from the depot; ml h_node add_problem "rx_road" "rx_trip" "Depot"; #3. Compare optimal and greedy or blind search strategies; json load "rx_algos" "[\"dijkstra\", \"a_star\", \"bellman_ford\", \"i_d_a_star\", \"breadth_first_search\", \"depth_first_search\", \"hill_climbing\"]"; -> "[out]" ""; for (-> "[i]" 0) (< [i] (json count "rx_algos" /)) (++ "[i]") { -> "[algo]" (json select "rx_algos" (concat "/[" [i] "]")); json load "rx_res" (ml h_node predict "rx_road" "rx_trip" [algo] "Store" (if (equal [algo] "hill_climbing") {true} {null})); -> "[out]" (concat [out] [algo] "=" (json select "rx_res" "/cost") " min "); }; -> "[best]" (ml h_node predict "rx_road" "rx_trip" "dijkstra" "Store" null); json load "rx_res" [best]; -> "[path]" "Depot"; for (-> "[i]" 1) (< [i] (json count "rx_res" "/optimalPaths/[0]")) (++ "[i]") { -> "[path]" (concat [path] " > " (json select "rx_res" (concat "/optimalPaths/[0]/[" [i] "]"))); }; -> "[open]" (json count "rx_algos" /); if (== (ml h_node exist_problem "rx_road" "rx_trip") 1) {json load "rx_pb" (ml h_node show_problem "rx_road"); -> "[open]" (json count "rx_pb" /); json unload "rx_pb";}; #4. Free the memory; ml h_node close_problem "rx_road" "rx_trip"; ml h_node close "rx_road"; json unload "rx_algos"; json unload "rx_res"; concat [out] "| best route: " [path] " | open problems before cleanup: " [open] " | graph rx_road still loaded: " (ml h_node exist "rx_road") ", all graphs: " (ml h_node show)
dijkstra=28.0 min a_star=28.0 min bellman_ford=28.0 min i_d_a_star=28.0 min breadth_first_search=32.0 min depth_first_search=31.0 min hill_climbing=32.0 min | best route: Depot > B > D > Store | open problems before cleanup: 1 | graph rx_road still loaded: 0, all graphs: []
- The cost of BFS and DFS shows why they are not optimal on weighted graphs.
- Load an undirected network by passing false as the second parameter.
Data quality gate before training
Before training a model on CRM rows, check each cell against the rules of the MentDB data quality catalogue (e-mail, integer, valid date) and list the rejected cells and the rows kept.
#1. Raw training rows exported from the CRM; json load "rx_rows" "[{\"email\": \"ana@shop.com\", \"age\": \"34\", \"signup\": \"2026-03-14\"}, {\"email\": \"bob@shop\", \"age\": \"41\", \"signup\": \"2026-02-30\"}, {\"email\": \"\", \"age\": \"2x\", \"signup\": \"2026-05-02\"}, {\"email\": \"li@mail.org\", \"age\": \"27\", \"signup\": \"2026-06-21\"}, {\"email\": \"zoe@mail.org\", \"age\": \"-\", \"signup\": \"21/06/2026\"}]"; #2. Rule per column, taken from the data quality catalogue; json load "rx_rules" "{\"email\": \"is_email\", \"age\": \"is_integer\", \"signup\": \"is_date\"}"; json load "rx_fields" (json fields "rx_rules" /); json load "rx_cat" (dq algorithm show); json load "rx_report" "{}"; json iobject "rx_report" / "rules_in_catalogue" (json count "rx_cat" /) NUM; #3. Run each rule on each cell, the rule reads the value in [VAR]; -> "[keep]" 0; for (-> "[r]" 0) (< [r] (json count "rx_rows" /)) (++ "[r]") { -> "[ok]" 1; for (-> "[f]" 0) (< [f] (json count "rx_fields" /)) (++ "[f]") { -> "[field]" (json select "rx_fields" (concat "/[" [f] "]")); -> "[algo]" (json select "rx_rules" (concat "/" [field])); if (== (dq algorithm exist [algo]) 0) {exception (1) (concat "Unknown rule " [algo])}; -> "[VAR]" (json select "rx_rows" (concat "/[" [r] "]/" [field])); if (== (eval (dq algorithm get [algo])) 0) {-> "[ok]" 0; json iobject "rx_report" / (concat "row" [r] "_" [field]) [VAR] STR;}; }; -> "[keep]" (+ [keep] [ok]); }; json iobject "rx_report" / "rows_kept" [keep] NUM; json iobject "rx_report" / "rows_rejected" (- (json count "rx_rows" /) [keep]) NUM; json unload "rx_rows"; json unload "rx_rules"; json unload "rx_fields"; json unload "rx_cat"; json doc "rx_report"
{
"row2_email": "",
"rules_in_catalogue": 24.0,
"row1_email": "bob@shop",
"rows_rejected": 3.0,
"row4_age": "-",
"row4_signup": "21/06/2026",
"row1_signup": "2026-02-30",
"row2_age": "2x",
"rows_kept": 2.0
}- Add your own rule with
dq algorithm set(see the next recipe). - On a real table,
dq generatewrites the full audit script for a column.
Audit a database column with a custom rule
Add a company rule for SKU codes, generate the audit script of a MySQL column, run the analysis and drill down on the rows that break the rule. Needs a database connection (demo_cm_mysql), so it is not executed here.
#1. Add a company rule to the catalogue: SKU codes like AB-12345; dq algorithm set "is_sku" (mql { type is_matches_regex [VAR] "[A-Z]{2}-[0-9]{5}"; }); dq algorithm exist "is_sku"; dq algorithm get "is_sku"; #2. Generate the audit script of the column sku of the products table; -> "[script]" (dq generate "demo_cm_mysql" "products" "sku" "[\"is_sku\", \"is_null_or_empty\"]" "select * from products limit 0, 500"); #3. The rule was only needed to generate the script; dq algorithm remove "is_sku"; #4. Run the analysis: the script ends with dq analyse; eval [script]; #5. Drill down on the rows that break the SKU rule; dq analyse show "demo_cm_mysql" (json doc "dq") "is_sku" "sku" "products" "select * from products limit 0, 500";
dq algorithm existanddq algorithm getcheck the rule before using it.- Remove temporary rules with
dq algorithm removeto keep the catalogue clean.
Customer segments straight from a SQL table
The customer features live in a database table. The script loads them into DBSCAN with one SQL query (radius 8, at least 2 neighbours), then counts the segments, the customers per segment and the noise points.
#1. A local H2 database plays the customer warehouse; cm set "tuto_p4h_crm" {execute "db.h2.embedded.config.get" "[database]" "./tmp/tuto_p4h_db/crm" "[user]" "sa" "[password]" "sa" ;}; sql connect "rx_p4h_crm" {cm get "tuto_p4h_crm";}; sql dml "rx_p4h_crm" "create table customers (id int primary key, orders_year double, basket_eur double)"; sql dml "rx_p4h_crm" "insert into customers values (1, 2, 35), (2, 3, 38), (3, 2, 41), (4, 3, 36), (5, 12, 62), (6, 13, 60), (7, 11, 65), (8, 12, 58), (9, 25, 140), (10, 26, 135), (11, 24, 142), (12, 40, 20)"; #2. DBSCAN straight from the SQL query: radius 8, at least 2 neighbours; ml cluster load "rx_p4h_seg" "tuto_p4h_crm" "orders_year" "basket_eur" 8 2 "select * from customers"; -> "[nb]" (ml cluster nb "rx_p4h_seg"); -> "[sizes]" ""; for (-> "[i]" 0) (< [i] [nb]) (++ "[i]") { -> "[sizes]" (concat [sizes] " " (ml cluster nb_point "rx_p4h_seg" [i])); }; -> "[loaded]" (string contains (ml cluster show) "rx_p4h_seg"); #3. Clean up: model, table, connection, files; ml cluster close "rx_p4h_seg"; sql dml "rx_p4h_crm" "drop table customers"; sql disconnect "rx_p4h_crm"; cm remove "tuto_p4h_crm"; file delete "tmp/tuto_p4h_db/crm.mv.db"; file delete "tmp/tuto_p4h_db"; concat "model listed in ml cluster show: " [loaded] " | segments: " [nb] " | customers per segment:" [sizes] " | noise: " (- 12 (math eval (concat "0" (string replace [sizes] " " "+")) "{}"))
model listed in ml cluster show: 1 | segments: 3 | customers per segment: 4 4 3 | noise: 1
- Point
cm setto your real warehouse (MySQL, PostgreSQL...) and keep the sameml cluster loadline. - Draw the segments with
ml cluster xy_scatter. - Store the segment of each customer back with
sql dmlupdate.
Linear regression solved by QR decomposition
A house price model price = b0 + b1.area + b2.rooms is fitted on 6 sales. The script solves the least squares with a Householder QR (numerically stable), checks it against the built-in solver and gives the condition number of X.
#1. Design matrix X = [1, area m2, rooms] and the prices in kEUR; -> "[X]" "[[1, 52, 2], [1, 75, 3], [1, 98, 4], [1, 60, 2], [1, 120, 5], [1, 85, 3]]"; -> "[y]" "[210, 290, 365, 228, 450, 318]"; #2. Householder QR: X = Q.R, then R1.beta = Q1t.y on the first 3 rows; json load "rx_p4h_qr" (matrix qr [X]); -> "[qty]" (matrix mul (matrix transpose (json select "rx_p4h_qr" "/Q")) [y]); json load "rx_p4h_r" (json select "rx_p4h_qr" "/R"); json load "rx_p4h_b" [qty]; -> "[R1]" (concat "[" (json select "rx_p4h_r" "/[0]") "," (json select "rx_p4h_r" "/[1]") "," (json select "rx_p4h_r" "/[2]") "]"); -> "[b1]" (concat "[" (json select "rx_p4h_b" "/[0]") "," (json select "rx_p4h_b" "/[1]") "," (json select "rx_p4h_b" "/[2]") "]"); -> "[beta]" (matrix solve [R1] [b1]); #3. Check against the direct least squares solver and get the condition number; concat "beta by QR: " [beta] " | beta by matrix solve: " (matrix solve [X] [y]) " | cond X: " (math round (matrix cond [X]) 1)
beta by QR: [30.523408732246203,2.7154129405576017,18.279326670173575] | beta by matrix solve: [30.52340873224634,2.715412940557595,18.279326670173713] | cond X: 566.6
- A large
matrix condmeans collinear features: standardise them or add a ridge term. - Get the residuals with
matrix mulX.beta thenmatrix sub. - Compare with
pa rm load_from_jsonandpa rm estimate_regression_parameters.
Is the new model really better? Mann-Whitney on CV folds
Two models (gradient boosting and random forest) were scored on 7 cross-validation folds (MAE). The script runs a non-parametric Mann-Whitney U test, the difference of medians and the spread, then gives a decision.
#1. MAE of two models on 7 cross-validation folds; -> "[gbm]" "[3.1, 2.8, 3.4, 2.9, 3.0, 2.7, 3.2]"; -> "[rf]" "[3.6, 3.3, 3.9, 3.5, 3.1, 3.8, 3.4]"; #2. Non-parametric test: the fold errors are not assumed normal; json load "rx_p4h_mw" (stat mann_whitney [gbm] [rf]); -> "[p]" (json select "rx_p4h_mw" "/p_value"); #3. Effect size: difference of medians and spread; -> "[dmed]" (- (stat median [rf]) (stat median [gbm])); concat "U=" (json select "rx_p4h_mw" "/u") " p=" (math round [p] 4) " (" (json select "rx_p4h_mw" "/method") ") | median gain " (math round [dmed] 2) " | sd gbm " (math round (stat stddev [gbm]) 3) " sd rf " (math round (stat stddev [rf]) 3) " | " (if (< [p] 0.05) {"keep the gradient boosting"} {"no significant difference"})
U=4 p=0.0104 (asymptotic) | median gain 0.5 | sd gbm 0.241 sd rf 0.279 | keep the gradient boosting
- When the folds are the same for both models, the paired
stat wilcoxonis more powerful. - Use
stat ttest_2when the errors are close to normal (stat shapiro). - With more than two models,
stat kruskalcompares them all at once.
Class imbalance audit and balanced class weights
Before training a fraud classifier, the script builds the frequency table of the labels, finds the majority class, the imbalance ratio and the balanced class weights w = N / (k . count) to pass to the loss function.
#1. Labels of the training set: 0 = legit, 1 = fraud, 2 = chargeback; -> "[labels]" "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,2,0,0,0,0,0,0,0,0,1,0]"; #2. Frequency table and majority class; json load "rx_p4h_freq" (stat frequency [labels]); -> "[n]" (json count "rx_p4h_freq" /); -> "[weights]" ""; #3. Balanced class weights: w = N / (k * count); for (-> "[i]" 0) (< [i] [n]) (++ "[i]") { -> "[c]" (json select "rx_p4h_freq" (concat "/[" [i] "]/count")); -> "[weights]" (concat [weights] " class " (json select "rx_p4h_freq" (concat "/[" [i] "]/value")) "=" (math round (math calc "30/([n]*[c])") 2)); }; -> "[cmax]" (json select "rx_p4h_freq" "/[0]/count"); -> "[cmin]" (json select "rx_p4h_freq" (concat "/[" (- [n] 1) "]/count")); concat "majority class: " (stat mode [labels]) " | imbalance ratio " (/ [cmax] [cmin]) ":1 | weights:" [weights]
majority class: [0] | imbalance ratio 25:1 | weights: class 0=0.4 class 1=2.5 class 2=10
- Above 10:1, prefer the precision-recall curve to the accuracy.
- Oversample the minority class with
stat dist_randomnoise around its points. - Check that the train and test splits keep the same frequencies with
stat chi2_test.
PCA: how much variance does the first component keep?
Three correlated sensor features are standardised, their correlation matrix is built and diagonalised. The script gives the eigenvalues, the share of variance of PC1 and checks the decomposition C = V.D.Vt.
#1. Three sensor features, standardised column by column; -> "[f1]" (stat zscore "[2.5, 0.5, 2.2, 1.9, 3.1, 2.3, 2.0, 1.0, 1.5, 1.1]"); -> "[f2]" (stat zscore "[2.4, 0.7, 2.9, 2.2, 3.0, 2.7, 1.6, 1.1, 1.6, 0.9]"); -> "[f3]" (stat zscore "[1.2, 0.3, 1.0, 1.1, 1.4, 0.9, 1.3, 0.2, 0.8, 0.4]"); #2. Correlation matrix of the standardised data; -> "[c12]" (stat covariance [f1] [f2]); -> "[c13]" (stat covariance [f1] [f3]); -> "[c23]" (stat covariance [f2] [f3]); -> "[C]" (concat "[[1," [c12] "," [c13] "],[" [c12] ",1," [c23] "],[" [c13] "," [c23] ",1]]"); #3. Eigen decomposition, check C = V D Vt with a diagonal matrix; json load "rx_p4h_eig" (matrix eigen [C]); -> "[vals]" (json select "rx_p4h_eig" "/values"); -> "[Vt]" (json select "rx_p4h_eig" "/vectors"); -> "[rebuilt]" (matrix mul (matrix mul (matrix transpose [Vt]) (matrix diag [vals])) [Vt]); -> "[pc1]" (math round (* 100 (/ (json select "rx_p4h_eig" "/values/[0]") (stat sum [vals]))) 1); concat "eigenvalues " [vals] " | PC1 explains " [pc1] "% of the variance | rebuild error " (math round (matrix norm_frobenius (matrix sub [rebuilt] [C])) 12)
eigenvalues [ 2.7428778970219247, 0.216649742645597, 0.040472360332478426 ] | PC1 explains 91.4% of the variance | rebuild error 0
- Keep the components until 95% of the variance (cumulative sum of the eigenvalues).
- Project the data on PC1 with
matrix multo get one compact feature. matrix svdgives the same axes without building the correlation matrix.
Random forest with a hold-out test set
Split 24 iris flowers into a stratified training set and a 25 % test set, train a random forest of 100 trees, then report the out-of-bag accuracy, the test accuracy, the macro F1 score and the most important measure.
#1. Iris measures: sepal length, sepal width, petal length, petal width - species 0, 1, 2; -> "[X]" "[[5.1, 3.5, 1.4, 0.2], [4.9, 3.0, 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [5.0, 3.6, 1.4, 0.2], [5.4, 3.9, 1.7, 0.4], [4.6, 3.4, 1.4, 0.3], [5.0, 3.4, 1.5, 0.2], [4.4, 2.9, 1.4, 0.2], [7.0, 3.2, 4.7, 1.4], [6.4, 3.2, 4.5, 1.5], [6.9, 3.1, 4.9, 1.5], [5.5, 2.3, 4.0, 1.3], [6.5, 2.8, 4.6, 1.5], [5.7, 2.8, 4.5, 1.3], [6.3, 3.3, 4.7, 1.6], [4.9, 2.4, 3.3, 1.0], [6.3, 3.3, 6.0, 2.5], [5.8, 2.7, 5.1, 1.9], [7.1, 3.0, 5.9, 2.1], [6.3, 2.9, 5.6, 1.8], [6.5, 3.0, 5.8, 2.2], [7.6, 3.0, 6.6, 2.1], [4.9, 2.5, 4.5, 1.7], [7.3, 2.9, 6.3, 1.8]]"; -> "[y]" "[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]"; #2. Stratified split, 25 % of each species for the test; json load "rx_sp" (ml prep split_stratified [X] [y] 0.25 7); #3. Random forest trained on the training set only; json load "rx_rf" (ml train "rx_rf" "random_forest_classifier" (json select "rx_sp" "/X_train") (json select "rx_sp" "/y_train") "{\"n_trees\": 100, \"seed\": 7}"); #4. Evaluation on the unseen flowers and importance of the measures; json load "rx_ev" (ml model evaluate "rx_rf" (json select "rx_sp" "/X_test") (json select "rx_sp" "/y_test")); json load "rx_imp" (ml model importance "rx_rf"); json load "rx_n" "[\"sepal length\", \"sepal width\", \"petal length\", \"petal width\"]"; -> "[best]" (json select "rx_n" (concat "/[" (json select "rx_imp" "/ranking/[0]") "]")); -> "[res]" (concat "OOB accuracy " (math round (json select "rx_rf" "/oob_accuracy") 3) " | test accuracy " (math round (json select "rx_ev" "/accuracy") 3) " | macro F1 " (math round (json select "rx_ev" "/macro/f1") 3) " | most important: " [best] " " (math round (json select "rx_imp" (concat "/importance/[" (json select "rx_imp" "/ranking/[0]") "]")) 3)); json unload "rx_sp"; json unload "rx_rf"; json unload "rx_ev"; json unload "rx_imp"; json unload "rx_n"; ml model delete "rx_rf"; [res]
OOB accuracy 1 | test accuracy 0.833 | macro F1 0.822 | most important: petal width 0.423
- Read the confusion matrix with
ml model evaluate(key confusion) to see which species are mixed up. - Replace the forest by
ml trainwith gradient_boosting_classifier and compare the test scores.
Benchmark five classifiers with a cross validation
Compare logistic regression, naive Bayes, k nearest neighbors, random forest and SVM on the same flowers with a 4-fold cross validation, then keep the algorithm with the best mean accuracy.
#1. The flowers; -> "[X]" "[[5.1, 3.5, 1.4, 0.2], [4.9, 3.0, 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [5.0, 3.6, 1.4, 0.2], [5.4, 3.9, 1.7, 0.4], [4.6, 3.4, 1.4, 0.3], [5.0, 3.4, 1.5, 0.2], [4.4, 2.9, 1.4, 0.2], [7.0, 3.2, 4.7, 1.4], [6.4, 3.2, 4.5, 1.5], [6.9, 3.1, 4.9, 1.5], [5.5, 2.3, 4.0, 1.3], [6.5, 2.8, 4.6, 1.5], [5.7, 2.8, 4.5, 1.3], [6.3, 3.3, 4.7, 1.6], [4.9, 2.4, 3.3, 1.0], [6.3, 3.3, 6.0, 2.5], [5.8, 2.7, 5.1, 1.9], [7.1, 3.0, 5.9, 2.1], [6.3, 2.9, 5.6, 1.8], [6.5, 3.0, 5.8, 2.2], [7.6, 3.0, 6.6, 2.1], [4.9, 2.5, 4.5, 1.7], [7.3, 2.9, 6.3, 1.8]]"; -> "[y]" "[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]"; #2. Each algorithm with its default parameters; json load "rx_alg" "[\"logistic\", \"naive_bayes\", \"knn_classifier\", \"random_forest_classifier\", \"svm_classifier\"]"; -> "[report]" ""; -> "[best]" ""; -> "[best_score]" -1; for (-> "[i]" 0) (< [i] (json count "rx_alg" "/")) (++ "[i]") { -> "[a]" (json select "rx_alg" (concat "/[" [i] "]")); json load "rx_cv" (ml cross_validate [a] [X] [y] 4 "{}" 1); -> "[m]" (math round (json select "rx_cv" "/mean") 3); -> "[report]" (concat [report] [a] " " [m] " +/- " (math round (json select "rx_cv" "/std") 3) "; "); if (> [m] [best_score]) {-> "[best_score]" [m]; -> "[best]" [a];}; }; json unload "rx_alg"; json unload "rx_cv"; concat [report] "best: " [best]
logistic 0.917 +/- 0.167; naive_bayes 0.917 +/- 0.096; knn_classifier 0.917 +/- 0.096; random_forest_classifier 0.958 +/- 0.083; svm_classifier 0.875 +/- 0.16; best: random_forest_classifier
- Tune the winner with
ml grid_searchbefore training the final model withml train. - Use
ml prep kfoldto get the fold indexes and compute your own metric on each fold.
Tune a gradient boosting with a grid search
Search the best number of trees and depth of a gradient boosting regressor predicting the yield of a process from temperature and pressure, then train the final model with the best parameters and predict two new settings.
#1. Process runs: temperature in C, pressure in bar - yield in %; -> "[X]" "[[150, 2], [160, 2], [170, 2], [180, 2], [150, 3], [160, 3], [170, 3], [180, 3], [150, 4], [160, 4], [170, 4], [180, 4], [155, 2.5], [165, 3.5], [175, 2.5], [175, 3.5]]"; -> "[y]" "[61.2, 66.0, 69.1, 70.2, 64.8, 70.1, 73.9, 74.6, 66.1, 71.8, 75.2, 75.9, 65.0, 72.4, 70.8, 74.9]"; #2. Grid search on n_estimators and max_depth; json load "rx_gs" (ml grid_search "gradient_boosting_regressor" [X] [y] 3 "{\"n_estimators\": [50, 150], \"max_depth\": [1, 2, 3]}" 1); #3. Final model with the best parameters; ml train "rx_gb" "gradient_boosting_regressor" [X] [y] (json select "rx_gs" "/best_params"); json load "rx_p" (ml model predict "rx_gb" "[[168, 3.2], [185, 4]]"); -> "[res]" (concat "best " (json select "rx_gs" "/best_params") " CV R2 " (math round (json select "rx_gs" "/best_score") 3) " | yield at 168 C / 3.2 bar: " (math round (json select "rx_p" "/[0]") 1) " % | at 185 C / 4 bar: " (math round (json select "rx_p" "/[1]") 1) " %"); json unload "rx_gs"; json unload "rx_p"; ml model delete "rx_gb"; [res]
best {
"max_depth": 1,
"n_estimators": 150
} CV R2 0.944 | yield at 168 C / 3.2 bar: 73.7 % | at 185 C / 4 bar: 75.8 %- A tree model does not extrapolate: outside the trained range the prediction stays flat, compare with
ml trainlinear. - Read the variable effects with
ml model importance.
Save a trained model and score with it later
Train a logistic regression on machine failures, save it in a file, remove it from memory as after a restart, reload it under a new id and score the failure probability of two machines.
#1. Machines: temperature in C, vibration in mm/s - failure 1 or 0; -> "[X]" "[[60, 2.1], [65, 2.5], [70, 3.0], [72, 2.2], [75, 3.8], [78, 4.1], [80, 3.5], [82, 4.6], [85, 5.0], [88, 4.4], [68, 4.0], [90, 5.5]]"; -> "[y]" "[0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1]"; ml train "rx_fail" "logistic" [X] [y]; #2. Save, delete, reload; ml model save "rx_fail" "tmp/rx_fail.model"; ml model delete "rx_fail"; ml model load "rx_fail_prod" "tmp/rx_fail.model"; #3. Probability of failure of two machines; json load "rx_info" (ml model info "rx_fail_prod"); json load "rx_pp" (ml model predict_proba "rx_fail_prod" "[[71, 3.1], [86, 4.9]]"); -> "[res]" (concat "P failure at 71 C / 3.1 mm/s: " (math round (json select "rx_pp" "/proba/[0]/[1]") 3) " | at 86 C / 4.9 mm/s: " (math round (json select "rx_pp" "/proba/[1]/[1]") 3) " | reloaded model: " (json select "rx_info" "/algo") " on " (json select "rx_info" "/n_samples") " machines"); json unload "rx_pp"; json unload "rx_info"; ml model delete "rx_fail_prod"; file delete "tmp/rx_fail.model"; [res]
P failure at 71 C / 3.1 mm/s: 0.072 | at 86 C / 4.9 mm/s: 0.991 | reloaded model: logistic on 12 machines
ml model showlists all the models in memory with their algorithm and size.- Load the models at server start in a script called by the boot sequence, then score in your web services.
Map the operating modes of a machine with a self-organizing map
Project 15 operating points of a pump (flow, head, power) on a 3 x 3 Kohonen map, read how many points fall on each node and on which node a new point lands.
#1. Operating points: flow m3/h, head m, power kW; -> "[X]" "[[20, 48, 5.1], [22, 47, 5.3], [21, 48, 5.2], [40, 41, 7.9], [42, 40, 8.1], [41, 40, 8.0], [60, 30, 10.2], [62, 29, 10.5], [61, 30, 10.3], [30, 45, 6.6], [50, 36, 9.1], [70, 24, 11.4], [72, 22, 11.6], [35, 43, 7.2], [55, 33, 9.7]]"; json load "rx_som" (ml fit "rx_som" "som" [X] "{\"rows\": 3, \"cols\": 3, \"epochs\": 200, \"seed\": 1}"); json load "rx_pos" (ml model transform "rx_som" "[[68, 25, 11.0]]"); -> "[res]" (concat "hits per node " (json select "rx_som" "/hits") " | quantization error " (math round (json select "rx_som" "/quantization_error") 3) " | new point at row " (json select "rx_pos" "/[0]/[0]") ", col " (json select "rx_pos" "/[0]/[1]")); json unload "rx_som"; json unload "rx_pos"; ml model delete "rx_som"; [res]
hits per node [ 3, 0, 2, 2, 1, 3, 3, 0, 1 ] | quantization error 0.161 | new point at row 0, col 2
- The weights key of
ml model infogives the prototype of each node in the original units. - Cluster the nodes with
ml hclustto merge them into a few operating modes.
Clusters of different densities with HDBSCAN
Group GPS positions of delivery stops where a dense city center and a spread suburb coexist: DBSCAN needs one radius for both, HDBSCAN finds both groups and isolates the outliers.
#1. Positions in km: city center, suburb, isolated stops; -> "[P]" "[[0, 0], [0.2, 0.1], [-0.1, 0.2], [0.1, -0.2], [0.2, 0.2], [-0.2, -0.1], [5, 5], [6.5, 4.8], [5.8, 6.4], [4.6, 6.1], [6.1, 5.6], [5.2, 4.1], [12, -3], [-6, 9]]"; json load "rx_db" (ml dbscan [P] 0.5 4 "euclidean"); json load "rx_h" (ml hdbscan [P] 4 4 "euclidean"); -> "[res]" (concat "DBSCAN radius 0.5: " (json select "rx_db" "/n_clusters") " cluster, " (json select "rx_db" "/n_noise") " noise | HDBSCAN: " (json select "rx_h" "/n_clusters") " clusters, sizes " (json select "rx_h" "/sizes") ", " (json select "rx_h" "/n_noise") " outliers"); json unload "rx_db"; json unload "rx_h"; [res]
DBSCAN radius 0.5: 1 cluster, 8 noise | HDBSCAN: 2 clusters, sizes [ 6, 6 ], 2 outliers
- The key probabilities of
ml hdbscantells how firmly each point belongs to its cluster. - A larger min_cluster_size merges the small groups into the bigger ones.
Predict the next step of a user journey
Learn the navigation paths of past visitors with a Compact Prediction Tree (CPT+) and predict the next page of three visitors in progress, with the share of each candidate.
#1. Past journeys; -> "[s]" "[[\"home\", \"search\", \"product\", \"cart\", \"pay\"], [\"home\", \"product\", \"cart\", \"pay\"], [\"search\", \"product\", \"cart\", \"pay\"], [\"home\", \"search\", \"product\", \"exit\"], [\"home\", \"faq\", \"contact\"], [\"faq\", \"contact\"], [\"home\", \"promo\", \"product\", \"cart\", \"pay\"], [\"search\", \"faq\", \"contact\"]]"; #2. Three journeys in progress; json load "rx_v" "[[\"product\", \"cart\"], [\"home\", \"faq\"], [\"home\", \"search\"]]"; -> "[res]" ""; for (-> "[i]" 0) (< [i] 3) (++ "[i]") { json load "rx_p" (ml assoc predict_next [s] (json select "rx_v" (concat "/[" [i] "]"))); -> "[res]" (concat [res] "visitor " (+ [i] 1) " -> " (json select "rx_p" "/prediction") " " (math round (* 100 (json select "rx_p" "/candidates/[0]/share")) 0) "% | "); }; json unload "rx_v"; json unload "rx_p"; [res]
visitor 1 -> pay 67% | visitor 2 -> contact 28% | visitor 3 -> product 27% |
ml assoc sequence_rulesexplains the predictions with readable rules.- The same method predicts the next failure code of a machine or the next purchase of a customer.
Stacking against the single models
Compare, by 5-fold cross validation, three single classifiers with their stacking (a logistic regression trained on their out-of-fold probabilities) and with a soft vote, before choosing the model to deploy.
#1. Rows and labels; -> "[X]" "[[-1.35, 0.45, 0.29], [1.44, -0.09, -0.08], [0.53, 1.32, -0.38], [0.58, 2.35, 0.69], [1.69, 1.18, 0.61], [1.7, 4.58, 2.45], [-0.86, -1.37, 0.44], [2.16, 1.09, 0.75], [0.3, 0.83, 0.63], [2.23, -0.65, -0.13], [-1.56, -0.21, -0.63], [2.0, 0.57, 0.48], [3.63, -0.06, 0.35], [0.54, 1.15, 0.54], [-0.1, 0.44, 0.04], [0.89, 2.18, 0.24], [2.06, -0.61, -0.07], [2.0, 2.64, 1.46], [1.1, -0.15, 0.58], [2.69, 1.67, 1.11], [0.84, 0.56, 0.98], [1.59, 2.17, 1.4], [0.31, 0.25, 0.12], [1.34, 0.28, -0.07], [0.65, 1.81, 0.88], [3.16, 2.43, 2.91], [0.2, 0.93, 0.95], [1.64, 0.67, 1.34], [0.27, 1.17, 0.54], [0.75, 4.42, 1.99], [0.19, 0.9, 0.12], [1.03, 1.37, 0.65], [1.23, 0.85, 0.38], [3.49, 0.59, 1.08], [1.22, -0.35, -1.04], [1.0, 1.78, 0.86], [1.7, -1.21, -0.68], [2.71, 1.31, 0.0], [0.89, 2.77, 1.32], [0.99, 1.01, 0.44], [2.74, 0.74, 0.73], [3.71, 1.4, 2.67], [-0.75, -0.57, 0.5], [3.57, 2.28, 2.87], [-2.0, -0.86, 0.27], [1.41, 2.2, 0.87], [0.87, 0.1, -0.14], [2.13, 2.01, 1.86], [-0.53, -0.28, -0.63], [3.76, 3.24, 3.98], [-0.53, -1.48, 0.35], [2.23, 4.12, 3.18], [-0.86, -2.93, 1.54], [2.21, -0.43, -0.23], [-1.77, 4.18, -1.9], [2.63, 1.29, 0.06], [-0.14, -1.87, 0.38], [-0.35, 0.41, 0.56], [1.81, -1.42, -1.62], [2.97, 2.52, 2.0]]"; -> "[y]" "[0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1]"; #2. The estimators of the ensembles; -> "[est]" "[\"naive_bayes\", {\"algo\": \"knn_classifier\", \"params\": {\"k\": 7}}, {\"algo\": \"decision_tree_classifier\", \"params\": {\"max_depth\": 3}}]"; json load "rx_a" (ml cross_validate "naive_bayes" [X] [y] 5 "{}" 42); json load "rx_b" (ml cross_validate "knn_classifier" [X] [y] 5 "{\"k\": 7}" 42); json load "rx_c" (ml cross_validate "decision_tree_classifier" [X] [y] 5 "{\"max_depth\": 3}" 42); -> "[res]" (concat "naive Bayes " (math round (json select "rx_a" "/mean") 3) " | k-NN " (math round (json select "rx_b" "/mean") 3) " | tree " (math round (json select "rx_c" "/mean") 3) " | "); json load "rx_cv" (ml cross_validate "stacking_classifier" [X] [y] 5 (concat "{\"estimators\": " [est] "}") 42); -> "[res]" (concat [res] "stacking " (math round (json select "rx_cv" "/mean") 3) " | "); json load "rx_cv" (ml cross_validate "voting_classifier" [X] [y] 5 (concat "{\"estimators\": " [est] "}") 42); -> "[res]" (concat [res] "soft vote " (math round (json select "rx_cv" "/mean") 3)); json unload "rx_a"; json unload "rx_b"; json unload "rx_c"; json unload "rx_cv"; [res]
naive Bayes 0.767 | k-NN 0.733 | tree 0.6 | stacking 0.767 | soft vote 0.7
bagging_classifierandadaboost_classifiertake any algorithm as base, ex: {"base": {"algo": "knn_classifier", "params": {"k": 3}}}.- Add {"passthrough": true} to give the original features to the final estimator too.
