diff --git a/CHANGELOG.md b/CHANGELOG.md index ce6b1ec..5fad6eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,10 +23,46 @@ numeric splits every other builder produces, and node arrays are calloc'd, so nothing existing changed shape. A level absent from the training data goes right at every node it reaches. -`RandomForest`, the Bagging estimators and the ExtraTree builders are -unchanged and remain numeric only. The canonical DecisionTree and -RandomForest digits rows are bit-identical, verified node by node rather -than by accuracy. +The canonical DecisionTree and RandomForest digits rows are +bit-identical, verified node by node rather than by accuracy. + +### Categorical splits in the ensemble builders (#457) + +The mask now reaches the ensembles that #354 left out. New entry points, +each taking the same `X.cols` mask on the same contract: + + random_forest_classifier_fit_categorical + random_forest_regressor_fit_categorical + bagging_classifier_fit_categorical + bagging_regressor_fit_categorical + extra_trees_classifier_fit_categorical + extra_trees_regressor_fit_categorical + decision_tree_classifier_fit_rf_categorical + extra_tree_classifier_fit_categorical + extra_tree_regressor_fit_categorical + +Every existing entry point calls its `_categorical` variant with an +all-zero mask, so there is one implementation per estimator and the +numeric path is the old code path. + +Inside the RandomForest builder the subset search is scoped by +`max_features` exactly as the threshold sweep is: a drawn categorical +feature gets its Breiman scan, an undrawn one is skipped. The feature +mask is drawn before any feature is looked at, so the PRNG consumption +does not depend on which features are categorical. + +ExtraTree splits a categorical feature on a random non-empty proper +subset of the levels present at the node, one coin flip per level, which +is the nominal rule from Geurts, Ernst and Wehenkel. `rand()` is +consumed only when the drawn feature is categorical. + +`ensemble.flow` evaluated every node as a threshold split at four inline +traversal sites. They now all go through one node-evaluation helper that +reads `cat_kind`. + +All 165 `RESULT` and `DETAIL` records from `benchmarks/bench_flow_v2.flow` +are unchanged, including the canonical RandomForest digits 0.955555558 +and the per-tree structural rows. ### Estimator hot-path optimization pass diff --git a/lib/scikit/ensemble.flow b/lib/scikit/ensemble.flow index db901d0..1a13160 100644 --- a/lib/scikit/ensemble.flow +++ b/lib/scikit/ensemble.flow @@ -18,6 +18,50 @@ extern { function srand(seed: u32) -> void } +# Which way one sample goes at one internal node. The single node-evaluation +# predicate for every tree traversal in this module. +# +# Issue #457: this module used to test `xv <= node.threshold` inline at four +# separate traversal sites, which was correct only for as long as no builder +# here set cat_kind. Once a forest, a Bagging estimator or an ExtraTree can +# produce a categorical node, each of those sites needs the set-membership +# test, and four copies of it is how the next inconsistency starts. Every +# traversal below calls this instead. +# +# cat_kind == 0 is a numeric node and takes the same `xv <= threshold` those +# sites always took, so a tree with no categorical split routes every row +# exactly as before, bit for bit. +# +# cat_kind == 1 is a categorical node: the level index is the feature value +# truncated to an integer, and the sample goes left when that level's bit is +# set in the 128-bit left set. Unseen levels go right, along with any value +# outside 0..127 or one that is not a whole number. +# +# This is deliberately the same predicate as `_tree_split_goes_left` in +# tree.flow. That one is not exported, and a non-exported function is not +# visible across modules in Flow, so the predicate cannot literally be shared; +# the two must be changed together. The name is module-unique because +# non-exported functions sharing a name across modules collide in the +# generated C (compiler bug #465). +function _ensemble_node_goes_left(node: ptr, xv: f32) -> bool { + if node.cat_kind == 0 { + return xv <= node.threshold + } + let c: i32 = xv as i32 + if (c as f32) != xv { return false } + if c < 0 || c >= 128 { return false } + let widx: i32 = c / 32 + let bit: i32 = c - widx * 32 + let mut w: u32 = node.cat_mask0 + if widx == 1 { w = node.cat_mask1 } + if widx == 2 { w = node.cat_mask2 } + if widx == 3 { w = node.cat_mask3 } + let one: u32 = 1 as u32 + let probe: u32 = one << (bit as u32) + let zero: u32 = 0 as u32 + return (w & probe) != zero +} + export struct RandomForestClassifier { trees: ptr, n_trees: i32, @@ -29,6 +73,28 @@ export struct RandomForestClassifier { } export function random_forest_classifier_fit(X: Matrix, y: ptr, n_classes: i32, n_trees: i32, max_depth: i32, seed: i32) -> RandomForestClassifier { + # Every feature numeric: calloc gives the all-zero mask the categorical fit + # reads as "no categorical features", so this is one implementation and the + # numeric forest is the old code path, draw for draw. + let none: ptr = calloc(X.cols as i64, 4) as ptr + let model: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, n_classes, n_trees, max_depth, seed, none) + free(none as ptr) + return model +} + +# Issue #457: a random forest whose flagged features are split on a subset of +# their levels rather than on a threshold. `categorical` is X.cols entries, +# non-zero meaning categorical, and a categorical column must hold whole +# numbers in 0..127 which are read as level indices. The 128-level cap is +# per feature, inherited from the node representation added in #441. +# +# The mask is passed straight down to each tree's builder. It changes nothing +# about the sampling: the bootstrap draws n_samples indices and then one seed +# for this tree's feature subsampling, in that order, exactly as before, and +# the per-node feature draw happens before any feature is looked at. So a +# numeric fit through this entry point and a numeric fit through +# random_forest_classifier_fit produce the same forest from the same seed. +export function random_forest_classifier_fit_categorical(X: Matrix, y: ptr, n_classes: i32, n_trees: i32, max_depth: i32, seed: i32, categorical: ptr) -> RandomForestClassifier { # Issue #398: this used to append at classes[n_found] with no check against # n_classes, which wrote past the allocation whenever y carried more # distinct labels than the caller declared. @@ -85,7 +151,7 @@ export function random_forest_classifier_fit(X: Matrix, y: ptr, n_classes: prng = PRNG { state: res.state } prng_state[0] = res.state - trees[t] = decision_tree_classifier_fit_rf(X_boot, y_boot, n_classes, max_depth, CRITERION_GINI, max_features, prng_state) + trees[t] = decision_tree_classifier_fit_rf_categorical(X_boot, y_boot, n_classes, max_depth, CRITERION_GINI, max_features, prng_state, categorical) } free(bootstrap_indices as ptr) @@ -150,7 +216,7 @@ function rf_clf_vote_counts(model: RandomForestClassifier, X: Matrix) -> Matrix let mut node_idx: i32 = 0 while not nodes[node_idx].is_leaf { let node: ptr = nodes + node_idx - if xd[row_base + node.feature] <= node.threshold { + if _ensemble_node_goes_left(node, xd[row_base + node.feature]) { node_idx = node.left } else { node_idx = node.right @@ -302,6 +368,18 @@ export struct RandomForestRegressor { } export function random_forest_regressor_fit(X: Matrix, y: ptr, n_trees: i32, max_depth: i32, seed: i32) -> RandomForestRegressor { + # All-zero mask: the categorical fit with no categorical feature is the old + # code path, and rand() is consumed in the same order. + let none: ptr = calloc(X.cols as i64, 4) as ptr + let model: RandomForestRegressor = random_forest_regressor_fit_categorical(X, y, n_trees, max_depth, seed, none) + free(none as ptr) + return model +} + +# Issue #457: the regression forest with a per-feature categorical mask. The +# per-tree builder is the plain regressor, whose subset search is Breiman's +# ordering by mean response and is exact for a squared-error criterion. +export function random_forest_regressor_fit_categorical(X: Matrix, y: ptr, n_trees: i32, max_depth: i32, seed: i32, categorical: ptr) -> RandomForestRegressor { let trees: ptr = malloc((n_trees as i64) * 128) as ptr srand(seed) @@ -321,7 +399,7 @@ export function random_forest_regressor_fit(X: Matrix, y: ptr, n_trees: i32 } } - trees[t] = decision_tree_regressor_fit(X_boot, y_boot, max_depth, CRITERION_MSE) + trees[t] = decision_tree_regressor_fit_categorical(X_boot, y_boot, max_depth, CRITERION_MSE, categorical) free(bootstrap_indices as ptr) matrix_free(X_boot) @@ -396,6 +474,19 @@ export struct BaggingClassifier { } export function bagging_classifier_fit(X: Matrix, y: ptr, n_classes: i32, n_estimators: i32, max_depth: i32, seed: i32) -> BaggingClassifier { + # All-zero mask: the categorical fit with no categorical feature is the old + # code path, and rand() is consumed in the same order. + let none: ptr = calloc(X.cols as i64, 4) as ptr + let model: BaggingClassifier = bagging_classifier_fit_categorical(X, y, n_classes, n_estimators, max_depth, seed, none) + free(none as ptr) + return model +} + +# Issue #457: bagged classification trees with a per-feature categorical mask, +# the same contract as decision_tree_classifier_fit_categorical. Each estimator +# is a full plain tree over a bootstrap sample, so every estimator sees the +# mask and can split on a subset of levels. +export function bagging_classifier_fit_categorical(X: Matrix, y: ptr, n_classes: i32, n_estimators: i32, max_depth: i32, seed: i32, categorical: ptr) -> BaggingClassifier { # Issue #398: this used to append at classes[n_found] with no check against # n_classes, which wrote past the allocation whenever y carried more # distinct labels than the caller declared. @@ -438,7 +529,7 @@ export function bagging_classifier_fit(X: Matrix, y: ptr, n_classes: i32, n dst = dst + n_cols } - trees[t] = decision_tree_classifier_fit(X_boot, y_boot, n_classes, max_depth, CRITERION_GINI) + trees[t] = decision_tree_classifier_fit_categorical(X_boot, y_boot, n_classes, max_depth, CRITERION_GINI, categorical) } free(indices as ptr) @@ -1201,7 +1292,7 @@ export function adaboost_regressor_predict(model: AdaBoostRegressor, X: Matrix) let mut node_idx: i32 = 0 while not nodes[node_idx].is_leaf { let node: ptr = nodes + node_idx - if xd[row_base + node.feature] <= node.threshold { + if _ensemble_node_goes_left(node, xd[row_base + node.feature]) { node_idx = node.left } else { node_idx = node.right @@ -1318,6 +1409,19 @@ export struct ExtraTreesClassifier { export function extra_trees_classifier_fit( X: Matrix, y: ptr, n_classes: i32, n_trees: i32, max_depth: i32, seed: i32 +) -> ExtraTreesClassifier { + # All-zero mask: the categorical fit with no categorical feature is the old + # code path, and rand() is consumed in the same order. + let none: ptr = calloc(X.cols as i64, 4) as ptr + let model: ExtraTreesClassifier = extra_trees_classifier_fit_categorical(X, y, n_classes, n_trees, max_depth, seed, none) + free(none as ptr) + return model +} + +# Issue #457: the ExtraTrees classifier with a per-feature categorical mask. +export function extra_trees_classifier_fit_categorical( + X: Matrix, y: ptr, n_classes: i32, + n_trees: i32, max_depth: i32, seed: i32, categorical: ptr ) -> ExtraTreesClassifier { # Issue #398: this used to append at classes[n_found] with no check against # n_classes, which wrote past the allocation whenever y carried more @@ -1348,7 +1452,7 @@ export function extra_trees_classifier_fit( y_boot[i] = y[idx] } - trees[t] = decision_tree_classifier_fit(X_boot, y_boot, n_classes, max_depth, CRITERION_GINI) + trees[t] = decision_tree_classifier_fit_categorical(X_boot, y_boot, n_classes, max_depth, CRITERION_GINI, categorical) matrix_free(X_boot) array_free_f32(y_boot) @@ -1425,6 +1529,19 @@ export struct ExtraTreesRegressor { export function extra_trees_regressor_fit( X: Matrix, y: ptr, n_trees: i32, max_depth: i32, seed: i32 +) -> ExtraTreesRegressor { + # All-zero mask: the categorical fit with no categorical feature is the old + # code path, and rand() is consumed in the same order. + let none: ptr = calloc(X.cols as i64, 4) as ptr + let model: ExtraTreesRegressor = extra_trees_regressor_fit_categorical(X, y, n_trees, max_depth, seed, none) + free(none as ptr) + return model +} + +# Issue #457: the ExtraTrees regressor with a per-feature categorical mask. +export function extra_trees_regressor_fit_categorical( + X: Matrix, y: ptr, + n_trees: i32, max_depth: i32, seed: i32, categorical: ptr ) -> ExtraTreesRegressor { let trees: ptr = malloc((n_trees as i64) * 128) as ptr @@ -1443,7 +1560,7 @@ export function extra_trees_regressor_fit( y_boot[i] = y[idx] } - trees[t] = decision_tree_regressor_fit(X_boot, y_boot, max_depth, CRITERION_MSE) + trees[t] = decision_tree_regressor_fit_categorical(X_boot, y_boot, max_depth, CRITERION_MSE, categorical) matrix_free(X_boot) array_free_f32(y_boot) @@ -2364,6 +2481,16 @@ export struct BaggingRegressor { } export function bagging_regressor_fit(X: Matrix, y: ptr, n_estimators: i32, max_depth: i32, seed: i32) -> BaggingRegressor { + # All-zero mask: the categorical fit with no categorical feature is the old + # code path, and rand() is consumed in the same order. + let none: ptr = calloc(X.cols as i64, 4) as ptr + let model: BaggingRegressor = bagging_regressor_fit_categorical(X, y, n_estimators, max_depth, seed, none) + free(none as ptr) + return model +} + +# Issue #457: bagged regression trees with a per-feature categorical mask. +export function bagging_regressor_fit_categorical(X: Matrix, y: ptr, n_estimators: i32, max_depth: i32, seed: i32, categorical: ptr) -> BaggingRegressor { let n: i32 = X.rows let n_features: i32 = X.cols let trees: ptr = malloc((n_estimators as i64) * 128) as ptr @@ -2392,7 +2519,7 @@ export function bagging_regressor_fit(X: Matrix, y: ptr, n_estimators: i32, y_boot[j] = y[idx] dst = dst + n_features } - trees[t] = decision_tree_regressor_fit(X_boot, y_boot, max_depth, 2) + trees[t] = decision_tree_regressor_fit_categorical(X_boot, y_boot, max_depth, 2, categorical) } matrix_free(X_boot) @@ -2439,7 +2566,7 @@ export function bagging_regressor_predict(model: BaggingRegressor, X: Matrix) -> let mut node_idx: i32 = 0 while not nodes[node_idx].is_leaf { let node: ptr = nodes + node_idx - if xd[row_base + node.feature] <= node.threshold { + if _ensemble_node_goes_left(node, xd[row_base + node.feature]) { node_idx = node.left } else { node_idx = node.right @@ -2712,7 +2839,7 @@ function _leaf_index(nodes: ptr, node_idx: i32, x: ptr, counter: counter[0] = counter[0] + 1 return idx } - if x[node.feature] <= node.threshold { + if _ensemble_node_goes_left(node, x[node.feature]) { return _leaf_index(nodes, node.left, x, counter) } return _leaf_index(nodes, node.right, x, counter) diff --git a/lib/scikit/tree.flow b/lib/scikit/tree.flow index b62fa45..f3ffc4d 100644 --- a/lib/scikit/tree.flow +++ b/lib/scikit/tree.flow @@ -410,11 +410,11 @@ export function decision_tree_classifier_fit(X: Matrix, y: ptr, n_classes: # ordering is recomputed from the response at every node rather than fixed # once by the encoder. # -# Only the plain classifier and the plain regressor accept a categorical mask. -# RandomForest, the Bagging estimators and the ExtraTree builders go through -# their own builders, which never set cat_kind, and several of them walk nodes -# inline rather than through predict_one. They are unchanged and still numeric -# only. +# Issue #457 extended the mask to the ensembles. RandomForest, Bagging, +# ExtraTrees and the ExtraTree builders each have a _categorical entry point on +# this same contract, and ensemble.flow evaluates cat_kind at every traversal. +# The one thing still numeric only is the boosting family: GradientBoosting, +# AdaBoost and the histogram estimators. export function decision_tree_classifier_fit_categorical(X: Matrix, y: ptr, n_classes: i32, max_depth: i32, criterion: i32, categorical: ptr) -> DecisionTreeClassifier { # Issue #398: this used to append at classes[n_found] with no check against # n_classes, which wrote past the allocation whenever y carried more @@ -859,6 +859,19 @@ function _build_classifier_tree( # Tree builder with feature subsampling for RandomForest. # At each node, only max_features randomly chosen features are considered. +# +# Issue #457: a drawn feature flagged categorical in cat_card gets Breiman's +# subset scan instead of the threshold sweep, and an undrawn one is skipped +# whether it is categorical or numeric. The categorical scan runs from the same +# feature loop, after the same feature_mask test, so max_features means the +# same thing for both kinds of feature. +# +# The draw order is untouched. The feature mask is drawn before any feature is +# looked at, and it is drawn from n_features and the PRNG state alone, so which +# features are sampled does not depend on what those features contain. +# +# With no categorical feature max_card is 0, every cat_card entry is 0, the +# categorical branch is unreachable and this is the numeric builder unchanged. function _build_classifier_tree_rf( nodes: ptr, n_nodes: ptr, @@ -872,7 +885,11 @@ function _build_classifier_tree_rf( classes: ptr, n_classes: i32, max_features: i32, - prng_state: ptr + prng_state: ptr, + cat_card: ptr, + max_card: i32, + cat_iscratch: ptr, + cat_mean: ptr ) -> i32 { let node_idx: i32 = n_nodes[0] n_nodes[0] = n_nodes[0] + 1 @@ -897,6 +914,17 @@ function _build_classifier_tree_rf( let mut best_feature: i32 = -1 let mut best_threshold: f32 = 0.0 let mut best_impurity: f32 = 9999999999.0 + let mut best_is_cat: bool = false + let mut best_n_left_cats: i32 = 0 + + # Views into the one categorical scratch block the fit allocated. All five + # are empty when max_card is 0, and nothing below reads them unless some + # cat_card[f] is positive, which cannot happen when max_card is 0. + let cat_count: ptr = cat_iscratch + let cat_ci_sum: ptr = cat_iscratch + max_card + let cat_ids: ptr = cat_iscratch + 2 * max_card + let best_cats: ptr = cat_iscratch + 3 * max_card + let cat_class: ptr = cat_iscratch + 4 * max_card # Row offsets into X.data, computed once per node and reused by every # candidate feature and by the final partition, instead of recomputing @@ -967,6 +995,82 @@ function _build_classifier_tree_rf( for f in 0 to n_features { if feature_mask[f] == 0 { continue } + let card: i32 = cat_card[f] + if card > 0 { + # Breiman's subset search, the same scan the plain builder runs. + # Bucket this node's rows by level, keeping each level's class + # histogram and the sum of the class indices of its samples. + for lv in 0 to card { + cat_count[lv] = 0 + cat_ci_sum[lv] = 0 + } + let cclen: i32 = card * n_classes + for k in 0 to cclen { cat_class[k] = 0 } + + for i in 0 to n_indices { + let lv: i32 = x_data[row_base[i] + f] as i32 + let cls: i32 = class_idx[i] + cat_count[lv] = cat_count[lv] + 1 + cat_ci_sum[lv] = cat_ci_sum[lv] + cls + cat_class[lv * n_classes + cls] = cat_class[lv * n_classes + cls] + 1 + } + + # Order the levels present here by their mean class index and score + # only the k-1 prefixes of that ordering, so a 40-level column + # costs 39 candidates rather than 2^39. + let mut n_cats: i32 = 0 + for lv in 0 to card { + if cat_count[lv] > 0 { + cat_ids[n_cats] = lv + cat_mean[n_cats] = (cat_ci_sum[lv] as f32) / (cat_count[lv] as f32) + n_cats = n_cats + 1 + } + } + if n_cats < 2 { continue } + _sort_pairs(cat_mean, cat_ids, n_cats) + + for pi in 0 to n_present { + let c: i32 = present[pi] + left_counts[c] = 0 + right_counts[c] = total_counts[c] + } + + let mut n_left_c: i32 = 0 + for j in 0 to n_cats - 1 { + let lv: i32 = cat_ids[j] + for pi in 0 to n_present { + let c: i32 = present[pi] + let hit: i32 = cat_class[lv * n_classes + c] + left_counts[c] = left_counts[c] + hit + right_counts[c] = right_counts[c] - hit + } + n_left_c = n_left_c + cat_count[lv] + let n_right_c: i32 = n_indices - n_left_c + if n_left_c < 1 || n_right_c < 1 { continue } + + let gini_left: f32 = _tree_gini_present(left_counts, present, n_present, n_left_c) + let gini_right: f32 = _tree_gini_present(right_counts, present, n_present, n_right_c) + let nf: f32 = n_indices as f32 + let weighted: f32 = (n_left_c as f32) / nf * gini_left + (n_right_c as f32) / nf * gini_right + + # The same strict comparison as the numeric scan below, run + # from the same feature loop in the same index order, so the + # tie-break rule pinned by #204 and #426 is unchanged: the + # lowest feature index wins among equals whether the winner is + # numeric or categorical, and within one categorical feature + # the shortest left set wins. + if weighted < best_impurity { + best_impurity = weighted + best_feature = f + best_threshold = 0.0 + best_is_cat = true + best_n_left_cats = j + 1 + for k in 0 to j + 1 { best_cats[k] = cat_ids[k] } + } + } + continue + } + # Gather and check for a constant column in one pass. If every sample # shares a value, the scan below hits `sort_values[i] == sort_values[i+1]` # at every position and evaluates no candidate at all, so skipping the @@ -1019,6 +1123,7 @@ function _build_classifier_tree_rf( best_impurity = weighted best_feature = f best_threshold = (sort_values[i] + sort_values[i + 1]) / 2.0 + best_is_cat = false } } } @@ -1053,13 +1158,30 @@ function _build_classifier_tree_rf( let mut left_n: i32 = 0 let mut right_n: i32 = 0 - for i in 0 to n_indices { - if x_data[row_base[i] + best_feature] <= best_threshold { - left_indices[left_n] = indices[i] - left_n = left_n + 1 - } else { - right_indices[right_n] = indices[i] - right_n = right_n + 1 + if best_is_cat { + # Commit the left set first so the routing here is the identical + # predicate predict will use on the same node, rather than a second + # copy of it that could drift. + _tree_write_cat_mask(nodes, node_idx, best_cats, best_n_left_cats) + let split_node: ptr = nodes + node_idx + for i in 0 to n_indices { + if _tree_split_goes_left(split_node, x_data[row_base[i] + best_feature]) { + left_indices[left_n] = indices[i] + left_n = left_n + 1 + } else { + right_indices[right_n] = indices[i] + right_n = right_n + 1 + } + } + } else { + for i in 0 to n_indices { + if x_data[row_base[i] + best_feature] <= best_threshold { + left_indices[left_n] = indices[i] + left_n = left_n + 1 + } else { + right_indices[right_n] = indices[i] + right_n = right_n + 1 + } } } @@ -1071,8 +1193,8 @@ function _build_classifier_tree_rf( nodes[node_idx].n_samples = n_indices nodes[node_idx].impurity = parent_impurity - let left_idx: i32 = _build_classifier_tree_rf(nodes, n_nodes, X, y, left_indices, left_n, depth + 1, max_depth, criterion, classes, n_classes, max_features, prng_state) - let right_idx: i32 = _build_classifier_tree_rf(nodes, n_nodes, X, y, right_indices, right_n, depth + 1, max_depth, criterion, classes, n_classes, max_features, prng_state) + let left_idx: i32 = _build_classifier_tree_rf(nodes, n_nodes, X, y, left_indices, left_n, depth + 1, max_depth, criterion, classes, n_classes, max_features, prng_state, cat_card, max_card, cat_iscratch, cat_mean) + let right_idx: i32 = _build_classifier_tree_rf(nodes, n_nodes, X, y, right_indices, right_n, depth + 1, max_depth, criterion, classes, n_classes, max_features, prng_state, cat_card, max_card, cat_iscratch, cat_mean) nodes[node_idx].left = left_idx nodes[node_idx].right = right_idx @@ -1085,6 +1207,25 @@ function _build_classifier_tree_rf( # Fit a decision tree with feature subsampling (for RandomForest). export function decision_tree_classifier_fit_rf(X: Matrix, y: ptr, n_classes: i32, max_depth: i32, criterion: i32, max_features: i32, prng_state: ptr) -> DecisionTreeClassifier { + # Every feature numeric: calloc gives the all-zero mask the categorical + # fit reads as "no categorical features", and with max_card 0 the builder + # never enters the categorical branch, so this is the old code path. + let none: ptr = calloc(X.cols as i64, 4) as ptr + let model: DecisionTreeClassifier = decision_tree_classifier_fit_rf_categorical(X, y, n_classes, max_depth, criterion, max_features, prng_state, none) + free(none as ptr) + return model +} + +# Issue #457: the feature-subsampling fit RandomForest uses, with the same +# per-feature categorical mask the plain classifier fit takes. `categorical` is +# X.cols entries, non-zero meaning categorical, and a categorical column must +# hold whole numbers in 0..127 which are read as level indices. +# +# The subset search is the same Breiman ordering the plain builder runs, and +# the same 128-level cap applies. It is scoped by max_features exactly as the +# threshold sweep is: a categorical feature that was not drawn at a node is not +# scanned there. +export function decision_tree_classifier_fit_rf_categorical(X: Matrix, y: ptr, n_classes: i32, max_depth: i32, criterion: i32, max_features: i32, prng_state: ptr, categorical: ptr) -> DecisionTreeClassifier { # Issue #398: this used to append at classes[n_found] with no check against # n_classes, which wrote past the allocation whenever y carried more # distinct labels than the caller declared. @@ -1099,7 +1240,20 @@ export function decision_tree_classifier_fit_rf(X: Matrix, y: ptr, n_classe let indices: ptr = malloc((X.rows as i64) * 4) as ptr for i in 0 to X.rows { indices[i] = i } - let _final: i32 = _build_classifier_tree_rf(nodes, n_nodes_ptr, X, y, indices, X.rows, 0, max_depth, criterion, classes, n_classes, max_features, prng_state) + # Categorical bookkeeping, allocated once per tree rather than per node so + # the numeric path picks up no extra allocation at all. With no categorical + # feature max_card is 0 and these are a 16-byte stub and one float. + let cat_card: ptr = _tree_cat_cardinalities(X, categorical) + let max_card: i32 = _tree_cat_max_card(cat_card, X.cols) + let ccap: i64 = max_card as i64 + let cat_iscratch: ptr = malloc((ccap * 4 + ccap * (n_classes as i64)) * 4 + 16) as ptr + let cat_mean: ptr = array_new_f32(max_card + 1) + + let _final: i32 = _build_classifier_tree_rf(nodes, n_nodes_ptr, X, y, indices, X.rows, 0, max_depth, criterion, classes, n_classes, max_features, prng_state, cat_card, max_card, cat_iscratch, cat_mean) + + free(cat_card as ptr) + free(cat_iscratch as ptr) + array_free_f32(cat_mean) free(indices as ptr) let n_nodes: i32 = n_nodes_ptr[0] @@ -1583,12 +1737,92 @@ export struct ExtraTreeClassifier { fitted: bool } +# Draw the left set for an ExtraTree split on a categorical feature. +# +# Issue #457. The ExtraTree split rule is "pick one feature at random, then one +# cut at random inside it". On an ordered feature the random cut is the +# midpoint of two randomly drawn sample values. The subset analogue, the one +# Geurts, Ernst and Wehenkel give for a nominal attribute, is a random +# non-empty proper subset of the levels present at the node, drawn by one coin +# flip per level. +# +# `seen`, `present_lv` and `left_lv` are 128-entry scratch arrays owned by the +# fit, so a node allocates nothing. The return value is the number of levels +# written to left_lv, and 0 means the node has fewer than two levels here and +# cannot be split on this feature at all. +# +# A draw that lands on the empty set or on every level is not a split, so it is +# repaired to the single lowest level rather than redrawn. Redrawing would make +# the number of rand() calls at a node depend on the draws themselves, and the +# repaired set is still a valid random subset of a set the node can separate. +# +# rand() is consumed only when this is reached, which is only when the node's +# drawn feature is categorical, so the numeric path draws exactly what it drew +# before: one feature and two rows per node, in that order. +function _tree_extra_cat_left_set(X: Matrix, n: i32, feature: i32, seen: ptr, present_lv: ptr, left_lv: ptr) -> i32 { + for lv in 0 to 128 { seen[lv] = 0 } + for i in 0 to n { + let v: f32 = matrix_at(X, i, feature) + let c: i32 = v as i32 + if (c as f32) == v { + if c >= 0 && c < 128 { seen[c] = 1 } + } + } + let mut k: i32 = 0 + for lv in 0 to 128 { + if seen[lv] == 1 { + present_lv[k] = lv + k = k + 1 + } + } + if k < 2 { return 0 } + + let mut nl: i32 = 0 + for j in 0 to k { + let coin: i32 = rand() % 2 + if coin == 1 { + left_lv[nl] = present_lv[j] + nl = nl + 1 + } + } + if nl == 0 || nl == k { + left_lv[0] = present_lv[0] + nl = 1 + } + return nl +} + export function extra_tree_classifier_fit(X: Matrix, y: ptr, n_samples: i32, n_features: i32, n_classes: i32, max_depth: i32, seed: i32) -> ExtraTreeClassifier { + # Every feature numeric: calloc gives the all-zero mask the categorical fit + # reads as "no categorical features", so this is the old code path. + let none: ptr = calloc(X.cols as i64, 4) as ptr + let model: ExtraTreeClassifier = extra_tree_classifier_fit_categorical(X, y, n_samples, n_features, n_classes, max_depth, seed, none) + free(none as ptr) + return model +} + +# Issue #457: the ExtraTree classifier with a per-feature categorical mask. +# `categorical` is X.cols entries, non-zero meaning categorical, and a +# categorical column must hold whole numbers in 0..127 read as level indices. +# A node whose drawn feature is categorical splits on a random subset of the +# levels present there instead of on a random threshold. +export function extra_tree_classifier_fit_categorical(X: Matrix, y: ptr, n_samples: i32, n_features: i32, n_classes: i32, max_depth: i32, seed: i32, categorical: ptr) -> ExtraTreeClassifier { srand(seed) let nodes: ptr = calloc(1024 as i64, 64) as ptr let mut n_nodes: i32 = 0 - n_nodes = _extra_tree_clf_build(nodes, n_nodes, X, y, n_samples, n_features, n_classes, 0, max_depth, seed) + # One allocation per fit, not per node, so the numeric path is unchanged. + let cat_card: ptr = _tree_cat_cardinalities(X, categorical) + let et_seen: ptr = malloc(512) as ptr + let et_present: ptr = malloc(512) as ptr + let et_left: ptr = malloc(512) as ptr + + n_nodes = _extra_tree_clf_build(nodes, n_nodes, X, y, n_samples, n_features, n_classes, 0, max_depth, seed, cat_card, et_seen, et_present, et_left) + + free(cat_card as ptr) + free(et_seen as ptr) + free(et_present as ptr) + free(et_left as ptr) return ExtraTreeClassifier { nodes: nodes, @@ -1600,7 +1834,7 @@ export function extra_tree_classifier_fit(X: Matrix, y: ptr, n_samples: i32 } } -function _extra_tree_clf_build(nodes: ptr, node_idx: i32, X: Matrix, y: ptr, n: i32, p: i32, n_classes: i32, depth: i32, max_depth: i32, seed: i32) -> i32 { +function _extra_tree_clf_build(nodes: ptr, node_idx: i32, X: Matrix, y: ptr, n: i32, p: i32, n_classes: i32, depth: i32, max_depth: i32, seed: i32, cat_card: ptr, et_seen: ptr, et_present: ptr, et_left: ptr) -> i32 { let mut idx: i32 = node_idx let mut class_counts: ptr = malloc((n_classes as i64) * 4) as ptr @@ -1634,7 +1868,18 @@ function _extra_tree_clf_build(nodes: ptr, node_idx: i32, X: Matrix, y let feature: i32 = rand() % p let r1: i32 = rand() % n let r2: i32 = rand() % n - if r1 == r2 { + + # A categorical feature splits on a random subset of its levels; the two + # row draws above are spent either way, so the numeric draw order stands. + let is_cat: bool = cat_card[feature] > 0 + let mut n_left_cats: i32 = 0 + if is_cat { + n_left_cats = _tree_extra_cat_left_set(X, n, feature, et_seen, et_present, et_left) + } + + let mut degenerate: bool = r1 == r2 + if is_cat { degenerate = n_left_cats == 0 } + if degenerate { nodes[idx].is_leaf = true nodes[idx].value = best_class as f32 nodes[idx].feature = -1 @@ -1644,7 +1889,15 @@ function _extra_tree_clf_build(nodes: ptr, node_idx: i32, X: Matrix, y free(class_counts as ptr) return idx + 1 } - let threshold: f32 = (matrix_at(X, r1, feature) + matrix_at(X, r2, feature)) / 2.0 + + let mut threshold: f32 = 0.0 + if is_cat { + # Commit the left set before partitioning so the routing here is the + # identical predicate predict will use on the same node. + _tree_write_cat_mask(nodes, idx, et_left, n_left_cats) + } else { + threshold = (matrix_at(X, r1, feature) + matrix_at(X, r2, feature)) / 2.0 + } let left_y: ptr = array_new_f32(n) let right_y: ptr = array_new_f32(n) @@ -1653,8 +1906,12 @@ function _extra_tree_clf_build(nodes: ptr, node_idx: i32, X: Matrix, y let mut n_left: i32 = 0 let mut n_right: i32 = 0 + let split_node: ptr = nodes + idx for i in 0 to n { - if matrix_at(X, i, feature) <= threshold { + let xv: f32 = matrix_at(X, i, feature) + let mut goes_left: bool = xv <= threshold + if is_cat { goes_left = _tree_split_goes_left(split_node, xv) } + if goes_left { for j in 0 to p { matrix_set(left_X, n_left, j, matrix_at(X, i, j)) } left_y[n_left] = y[i] n_left = n_left + 1 @@ -1677,6 +1934,13 @@ function _extra_tree_clf_build(nodes: ptr, node_idx: i32, X: Matrix, y nodes[idx].feature = -1 nodes[idx].left = -1 nodes[idx].right = -1 + # A leaf carries no split, so the left set written above is cleared + # rather than left behind for a node dump to trip over. + nodes[idx].cat_kind = 0 + nodes[idx].cat_mask0 = 0 as u32 + nodes[idx].cat_mask1 = 0 as u32 + nodes[idx].cat_mask2 = 0 as u32 + nodes[idx].cat_mask3 = 0 as u32 matrix_free(left_X) matrix_free(right_X) array_free_f32(left_y) @@ -1685,11 +1949,11 @@ function _extra_tree_clf_build(nodes: ptr, node_idx: i32, X: Matrix, y } let left_idx: i32 = idx + 1 - let next_idx: i32 = _extra_tree_clf_build(nodes, left_idx, left_X, left_y, n_left, p, n_classes, depth + 1, max_depth, seed + 1) + let next_idx: i32 = _extra_tree_clf_build(nodes, left_idx, left_X, left_y, n_left, p, n_classes, depth + 1, max_depth, seed + 1, cat_card, et_seen, et_present, et_left) nodes[idx].right = next_idx let right_idx: i32 = next_idx - let final_idx: i32 = _extra_tree_clf_build(nodes, right_idx, right_X, right_y, n_right, p, n_classes, depth + 1, max_depth, seed + 2) + let final_idx: i32 = _extra_tree_clf_build(nodes, right_idx, right_X, right_y, n_right, p, n_classes, depth + 1, max_depth, seed + 2, cat_card, et_seen, et_present, et_left) matrix_free(left_X) matrix_free(right_X) @@ -1706,10 +1970,11 @@ export function extra_tree_classifier_predict(model: ExtraTreeClassifier, X: Mat matrix_row(X, i, row) let mut node: i32 = 0 while not model.nodes[node].is_leaf { - if row[model.nodes[node].feature] <= model.nodes[node].threshold { - node = model.nodes[node].left + let np: ptr = model.nodes + node + if _tree_split_goes_left(np, row[np.feature]) { + node = np.left } else { - node = model.nodes[node].right + node = np.right } } result[i] = model.nodes[node].value @@ -1735,11 +2000,32 @@ export struct ExtraTreeRegressor { } export function extra_tree_regressor_fit(X: Matrix, y: ptr, n_samples: i32, n_features: i32, max_depth: i32, seed: i32) -> ExtraTreeRegressor { + # Every feature numeric: calloc gives the all-zero mask the categorical fit + # reads as "no categorical features", so this is the old code path. + let none: ptr = calloc(X.cols as i64, 4) as ptr + let model: ExtraTreeRegressor = extra_tree_regressor_fit_categorical(X, y, n_samples, n_features, max_depth, seed, none) + free(none as ptr) + return model +} + +# Issue #457: the ExtraTree regressor with a per-feature categorical mask, the +# same contract as extra_tree_classifier_fit_categorical. +export function extra_tree_regressor_fit_categorical(X: Matrix, y: ptr, n_samples: i32, n_features: i32, max_depth: i32, seed: i32, categorical: ptr) -> ExtraTreeRegressor { srand(seed) let nodes: ptr = calloc(1024 as i64, 64) as ptr let mut n_nodes: i32 = 0 - n_nodes = _extra_tree_reg_build(nodes, n_nodes, X, y, n_samples, n_features, 0, max_depth, seed) + let cat_card: ptr = _tree_cat_cardinalities(X, categorical) + let et_seen: ptr = malloc(512) as ptr + let et_present: ptr = malloc(512) as ptr + let et_left: ptr = malloc(512) as ptr + + n_nodes = _extra_tree_reg_build(nodes, n_nodes, X, y, n_samples, n_features, 0, max_depth, seed, cat_card, et_seen, et_present, et_left) + + free(cat_card as ptr) + free(et_seen as ptr) + free(et_present as ptr) + free(et_left as ptr) return ExtraTreeRegressor { nodes: nodes, @@ -1750,7 +2036,7 @@ export function extra_tree_regressor_fit(X: Matrix, y: ptr, n_samples: i32, } } -function _extra_tree_reg_build(nodes: ptr, node_idx: i32, X: Matrix, y: ptr, n: i32, p: i32, depth: i32, max_depth: i32, seed: i32) -> i32 { +function _extra_tree_reg_build(nodes: ptr, node_idx: i32, X: Matrix, y: ptr, n: i32, p: i32, depth: i32, max_depth: i32, seed: i32, cat_card: ptr, et_seen: ptr, et_present: ptr, et_left: ptr) -> i32 { let mut idx: i32 = node_idx let mut mean: f32 = 0.0 @@ -1770,7 +2056,18 @@ function _extra_tree_reg_build(nodes: ptr, node_idx: i32, X: Matrix, y let feature: i32 = rand() % p let r1: i32 = rand() % n let r2: i32 = rand() % n - if r1 == r2 { + + # A categorical feature splits on a random subset of its levels; the two + # row draws above are spent either way, so the numeric draw order stands. + let is_cat: bool = cat_card[feature] > 0 + let mut n_left_cats: i32 = 0 + if is_cat { + n_left_cats = _tree_extra_cat_left_set(X, n, feature, et_seen, et_present, et_left) + } + + let mut degenerate: bool = r1 == r2 + if is_cat { degenerate = n_left_cats == 0 } + if degenerate { nodes[idx].is_leaf = true nodes[idx].value = mean nodes[idx].feature = -1 @@ -1779,7 +2076,15 @@ function _extra_tree_reg_build(nodes: ptr, node_idx: i32, X: Matrix, y nodes[idx].right = -1 return idx + 1 } - let threshold: f32 = (matrix_at(X, r1, feature) + matrix_at(X, r2, feature)) / 2.0 + + let mut threshold: f32 = 0.0 + if is_cat { + # Commit the left set before partitioning so the routing here is the + # identical predicate predict will use on the same node. + _tree_write_cat_mask(nodes, idx, et_left, n_left_cats) + } else { + threshold = (matrix_at(X, r1, feature) + matrix_at(X, r2, feature)) / 2.0 + } let left_y: ptr = array_new_f32(n) let right_y: ptr = array_new_f32(n) @@ -1788,8 +2093,12 @@ function _extra_tree_reg_build(nodes: ptr, node_idx: i32, X: Matrix, y let mut n_left: i32 = 0 let mut n_right: i32 = 0 + let split_node: ptr = nodes + idx for i in 0 to n { - if matrix_at(X, i, feature) <= threshold { + let xv: f32 = matrix_at(X, i, feature) + let mut goes_left: bool = xv <= threshold + if is_cat { goes_left = _tree_split_goes_left(split_node, xv) } + if goes_left { for j in 0 to p { matrix_set(left_X, n_left, j, matrix_at(X, i, j)) } left_y[n_left] = y[i] n_left = n_left + 1 @@ -1807,6 +2116,12 @@ function _extra_tree_reg_build(nodes: ptr, node_idx: i32, X: Matrix, y nodes[idx].threshold = 0.0 nodes[idx].left = -1 nodes[idx].right = -1 + # A leaf carries no split, so the left set written above is cleared. + nodes[idx].cat_kind = 0 + nodes[idx].cat_mask0 = 0 as u32 + nodes[idx].cat_mask1 = 0 as u32 + nodes[idx].cat_mask2 = 0 as u32 + nodes[idx].cat_mask3 = 0 as u32 matrix_free(left_X) matrix_free(right_X) array_free_f32(left_y) @@ -1820,11 +2135,11 @@ function _extra_tree_reg_build(nodes: ptr, node_idx: i32, X: Matrix, y nodes[idx].left = idx + 1 let left_idx: i32 = idx + 1 - let next_idx: i32 = _extra_tree_reg_build(nodes, left_idx, left_X, left_y, n_left, p, depth + 1, max_depth, seed + 1) + let next_idx: i32 = _extra_tree_reg_build(nodes, left_idx, left_X, left_y, n_left, p, depth + 1, max_depth, seed + 1, cat_card, et_seen, et_present, et_left) nodes[idx].right = next_idx let right_idx: i32 = next_idx - let final_idx: i32 = _extra_tree_reg_build(nodes, right_idx, right_X, right_y, n_right, p, depth + 1, max_depth, seed + 2) + let final_idx: i32 = _extra_tree_reg_build(nodes, right_idx, right_X, right_y, n_right, p, depth + 1, max_depth, seed + 2, cat_card, et_seen, et_present, et_left) matrix_free(left_X) matrix_free(right_X) @@ -1841,10 +2156,11 @@ export function extra_tree_regressor_predict(model: ExtraTreeRegressor, X: Matri matrix_row(X, i, row) let mut node: i32 = 0 while not model.nodes[node].is_leaf { - if row[model.nodes[node].feature] <= model.nodes[node].threshold { - node = model.nodes[node].left + let np: ptr = model.nodes + node + if _tree_split_goes_left(np, row[np.feature]) { + node = np.left } else { - node = model.nodes[node].right + node = np.right } } result[i] = model.nodes[node].value diff --git a/tests/test_ensemble_categorical.flow b/tests/test_ensemble_categorical.flow new file mode 100644 index 0000000..3630b4d --- /dev/null +++ b/tests/test_ensemble_categorical.flow @@ -0,0 +1,624 @@ +# Issue #457: categorical splits reach the forest, Bagging and ExtraTree +# builders, and every traversal in ensemble.flow evaluates the node's kind. +# +# PR #441 put cat_kind and a 128-bit inline left set on TreeNode and taught the +# two plain builders Breiman's subset ordering. The ensemble builders never set +# cat_kind and the four inline traversals in ensemble.flow read every node as a +# threshold split, so a forest could not express a subset split at all. +# +# Every dataset here has the parity shape from #441's test, widened to sixteen +# levels: the informative column's levels cycle 0..15 and the label is the +# level's parity, so the optimum is the even levels against the odd ones and no +# threshold on the label encoding 0 < 1 < ... < 15 can express it. + +import "lib/scikit/matrix.flow" +import "lib/scikit/tree.flow" +import "lib/scikit/ensemble.flow" + +extern { + function printf(fmt: string, ...) -> i32 +} + +# Column 0 cycles 0..15 and the label is the level's parity, so the optimum is +# the eight even levels against the eight odd ones. A threshold on the label +# encoding cuts that order in one place, so at depth 3 it can reach eight +# contiguous pairs of levels and every one of them is half one class and half +# the other: 50 percent, whatever the tree does. One subset split at depth 1 +# separates them exactly. +# +# Columns 1 to 3 are block indices: they change once per full cycle of the +# levels, so each of them is constant across a whole 0..15 cycle and carries no +# information about the parity at all. They are here so the fit is a real +# search over four features rather than a one-column special case, and so that +# max_features = sqrt(4) = 2 makes the forest's per-node feature draw matter. +function _t457_make_x(reps: i32) -> Matrix { + let n: i32 = 16 * reps + let X: Matrix = matrix_new(n, 4) + for i in 0 to n { + let lvl: i32 = i % 16 + let blk: i32 = i / 16 + matrix_set(X, i, 0, lvl as f32) + matrix_set(X, i, 1, (blk % 7) as f32) + matrix_set(X, i, 2, (blk % 5) as f32) + matrix_set(X, i, 3, (blk % 3) as f32) + } + return X +} + +function _t457_make_y(reps: i32) -> ptr { + let n: i32 = 16 * reps + let y: ptr = array_new_f32(n) + for i in 0 to n { + let lvl: i32 = i % 16 + y[i] = (lvl % 2) as f32 + } + return y +} + +# Only column 0 is categorical. Columns 1 to 3 stay numeric, so what is under +# test is the mixed case rather than an all-categorical shortcut. +function _t457_mask_col0(cols: i32) -> ptr { + let m: ptr = malloc((cols as i64) * 4) as ptr + for f in 0 to cols { m[f] = 0 } + m[0] = 1 + return m +} + +function _t457_mask_none(cols: i32) -> ptr { + let m: ptr = malloc((cols as i64) * 4) as ptr + for f in 0 to cols { m[f] = 0 } + return m +} + +function _t457_accuracy(pred: ptr, y: ptr, n: i32) -> f32 { + let mut hits: i32 = 0 + for i in 0 to n { + if pred[i] == y[i] { hits = hits + 1 } + } + return (hits as f32) / (n as f32) +} + +# How many nodes across the whole forest carry a fitted left set. +function _t457_forest_mask_nodes(m: RandomForestClassifier) -> i32 { + let mut found: i32 = 0 + for t in 0 to m.n_trees { + let tree: DecisionTreeClassifier = m.trees[t] + for k in 0 to tree.n_nodes { + let nd: ptr = tree.nodes + k + if nd.cat_kind == 1 { found = found + 1 } + } + } + return found +} + +function _t457_bag_mask_nodes(m: BaggingClassifier) -> i32 { + let mut found: i32 = 0 + for t in 0 to m.n_estimators { + let tree: DecisionTreeClassifier = m.trees[t] + for k in 0 to tree.n_nodes { + let nd: ptr = tree.nodes + k + if nd.cat_kind == 1 { found = found + 1 } + } + } + return found +} + +# Every field of every node of every tree, both forests. +function _t457_forests_match(a: RandomForestClassifier, b: RandomForestClassifier) -> bool { + if a.n_trees != b.n_trees { return false } + if a.n_classes != b.n_classes { return false } + for c in 0 to a.n_classes { + if a.classes[c] != b.classes[c] { return false } + } + for t in 0 to a.n_trees { + let ta: DecisionTreeClassifier = a.trees[t] + let tb: DecisionTreeClassifier = b.trees[t] + if ta.n_nodes != tb.n_nodes { return false } + for k in 0 to ta.n_nodes { + let na: ptr = ta.nodes + k + let nb: ptr = tb.nodes + k + if na.feature != nb.feature { return false } + if na.threshold != nb.threshold { return false } + if na.left != nb.left { return false } + if na.right != nb.right { return false } + if na.is_leaf != nb.is_leaf { return false } + if na.value != nb.value { return false } + if na.n_samples != nb.n_samples { return false } + if na.impurity != nb.impurity { return false } + if na.cat_kind != nb.cat_kind { return false } + if na.cat_mask0 != nb.cat_mask0 { return false } + if na.cat_mask1 != nb.cat_mask1 { return false } + if na.cat_mask2 != nb.cat_mask2 { return false } + if na.cat_mask3 != nb.cat_mask3 { return false } + } + } + return true +} + +function _t457_total_nodes(m: RandomForestClassifier) -> i32 { + let mut total: i32 = 0 + for t in 0 to m.n_trees { total = total + m.trees[t].n_nodes } + return total +} + +# --------------------------------------------------------------------------- +# 1. A forest on a subset optimum beats the same forest on a label encoding. +# --------------------------------------------------------------------------- +function _t457_test_forest_beats_label_encoding() -> i32 { + println("random forest on a categorical feature") + let mut failed: i32 = 0 + + let reps: i32 = 16 + let n: i32 = 16 * reps + let X: Matrix = _t457_make_x(reps) + let y: ptr = _t457_make_y(reps) + + let cat: ptr = _t457_mask_col0(4) + let num: ptr = _t457_mask_none(4) + + # Depth 3 is the budget where the difference is unambiguous. One subset + # split separates the classes exactly, and every tree that draws column 0 + # anywhere in its three levels finds it. A threshold on the label encoding + # reaches at best eight contiguous pairs of levels, each half one class and + # half the other, so no depth short of four helps it at all. + let cm: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, 2, 25, 3, 7, cat) + let nm: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, 2, 25, 3, 7, num) + + let cp: ptr = random_forest_classifier_predict(cm, X) + let np: ptr = random_forest_classifier_predict(nm, X) + let ca: f32 = _t457_accuracy(cp, y, n) + let na: f32 = _t457_accuracy(np, y, n) + + if ca <= na { + printf(" FAIL: categorical forest %.4f did not beat label-encoded %.4f\n", ca, na) + failed = failed + 1 + } else { + printf(" OK: categorical forest %.4f against label-encoded %.4f\n", ca, na) + } + + if ca < 0.999 { + printf(" FAIL: the subset split is exact, so the categorical forest should be perfect, got %.4f\n", ca) + failed = failed + 1 + } + + # At least one tree must actually carry a fitted left set, otherwise the + # accuracy above could have come from somewhere else entirely. + let masks: i32 = _t457_forest_mask_nodes(cm) + if masks < 1 { + println(" FAIL: no node in the categorical forest carries a mask") + failed = failed + 1 + } else { + printf(" OK: %d mask nodes across the forest\n", masks) + } + + # The label-encoded forest must carry none, which is the numeric-path + # guarantee stated in the same breath. + let nmasks: i32 = _t457_forest_mask_nodes(nm) + if nmasks != 0 { + printf(" FAIL: the label-encoded forest carries %d mask nodes\n", nmasks) + failed = failed + 1 + } + + array_free_f32(cp) + array_free_f32(np) + random_forest_classifier_free(cm) + random_forest_classifier_free(nm) + free(cat as ptr) + free(num as ptr) + array_free_f32(y) + matrix_free(X) + return failed +} + +# --------------------------------------------------------------------------- +# 2. The mask reaches every tree of a BaggingClassifier too. +# --------------------------------------------------------------------------- +function _t457_test_bagging_beats_label_encoding() -> i32 { + println("bagging classifier on a categorical feature") + let mut failed: i32 = 0 + + let reps: i32 = 16 + let n: i32 = 16 * reps + let X: Matrix = _t457_make_x(reps) + let y: ptr = _t457_make_y(reps) + + let cat: ptr = _t457_mask_col0(4) + let num: ptr = _t457_mask_none(4) + + let cm: BaggingClassifier = bagging_classifier_fit_categorical(X, y, 2, 8, 1, 11, cat) + let nm: BaggingClassifier = bagging_classifier_fit_categorical(X, y, 2, 8, 1, 11, num) + + let cp: ptr = bagging_classifier_predict(cm, X) + let np: ptr = bagging_classifier_predict(nm, X) + let ca: f32 = _t457_accuracy(cp, y, n) + let na: f32 = _t457_accuracy(np, y, n) + + if ca <= na { + printf(" FAIL: categorical bagging %.4f did not beat label-encoded %.4f\n", ca, na) + failed = failed + 1 + } else { + printf(" OK: categorical bagging %.4f against label-encoded %.4f\n", ca, na) + } + + let masks: i32 = _t457_bag_mask_nodes(cm) + if masks < 1 { + println(" FAIL: no node in the bagged ensemble carries a mask") + failed = failed + 1 + } else { + printf(" OK: %d mask nodes across the bagged ensemble\n", masks) + } + + let nmasks: i32 = _t457_bag_mask_nodes(nm) + if nmasks != 0 { + printf(" FAIL: the label-encoded bagged ensemble carries %d mask nodes\n", nmasks) + failed = failed + 1 + } + + array_free_f32(cp) + array_free_f32(np) + bagging_classifier_free(cm) + bagging_classifier_free(nm) + free(cat as ptr) + free(num as ptr) + array_free_f32(y) + matrix_free(X) + return failed +} + +# --------------------------------------------------------------------------- +# 3. An all-zero mask is the plain numeric fit, node for node. +# +# This is the claim the whole threading rests on: the old entry point calls the +# new one with an all-zero mask, so there is one implementation and the numeric +# forest is unchanged. Comparing every field of every node is the only way to +# assert it; two forests can agree on 100 predictions and differ in structure. +# --------------------------------------------------------------------------- +function _t457_test_empty_mask_is_the_plain_fit() -> i32 { + println("an all-numeric mask reproduces the plain forest") + let mut failed: i32 = 0 + + let reps: i32 = 16 + let X: Matrix = _t457_make_x(reps) + let y: ptr = _t457_make_y(reps) + let num: ptr = _t457_mask_none(4) + + let plain: RandomForestClassifier = random_forest_classifier_fit(X, y, 2, 12, 4, 7) + let masked: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, 2, 12, 4, 7, num) + + if not _t457_forests_match(plain, masked) { + println(" FAIL: the all-zero mask changed the forest") + failed = failed + 1 + } else { + printf(" OK: %d nodes identical field by field\n", _t457_total_nodes(plain)) + } + + random_forest_classifier_free(plain) + random_forest_classifier_free(masked) + free(num as ptr) + array_free_f32(y) + matrix_free(X) + return failed +} + +# --------------------------------------------------------------------------- +# 4. Same seed, same forest, twice over, with the mask on. +# +# Threading a mask must not make the fit depend on anything but the data and +# the seed. Fitting three times and comparing all three catches a builder that +# reads scratch it forgot to clear as readily as one that consumes a draw it +# should not. +# --------------------------------------------------------------------------- +function _t457_test_seed_stability() -> i32 { + println("same seed, same forest") + let mut failed: i32 = 0 + + let reps: i32 = 16 + let X: Matrix = _t457_make_x(reps) + let y: ptr = _t457_make_y(reps) + let cat: ptr = _t457_mask_col0(4) + + let a: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, 2, 12, 4, 19, cat) + let b: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, 2, 12, 4, 19, cat) + let c: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, 2, 12, 4, 19, cat) + + if not _t457_forests_match(a, b) { + println(" FAIL: two fits at seed 19 differ") + failed = failed + 1 + } + if not _t457_forests_match(a, c) { + println(" FAIL: the third fit at seed 19 differs") + failed = failed + 1 + } + if failed == 0 { + printf(" OK: three fits at seed 19 agree on all %d nodes\n", _t457_total_nodes(a)) + } + + # A different seed must produce a different forest, otherwise the check + # above passes for the wrong reason. + let d: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, 2, 12, 4, 20, cat) + if _t457_forests_match(a, d) { + println(" FAIL: seed 19 and seed 20 produced the same forest, so the seed is not read") + failed = failed + 1 + } + + random_forest_classifier_free(a) + random_forest_classifier_free(b) + random_forest_classifier_free(c) + random_forest_classifier_free(d) + free(cat as ptr) + array_free_f32(y) + matrix_free(X) + return failed +} + +# --------------------------------------------------------------------------- +# 5. Unseen levels go right at every node of every tree. +# +# The direction is defined rather than an error: a level absent from training +# has no bit set in any node. Asserting it at the forest is not the same as +# asserting it at one tree, because a vote can hide a wrong turn. +# --------------------------------------------------------------------------- +function _t457_test_unseen_level_predicts() -> i32 { + println("an unseen level takes the right branch through the forest") + let mut failed: i32 = 0 + + let reps: i32 = 16 + let X: Matrix = _t457_make_x(reps) + let y: ptr = _t457_make_y(reps) + let cat: ptr = _t457_mask_col0(4) + + let m: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, 2, 12, 3, 7, cat) + + # Level 9 never occurred in training. Every row here is otherwise valid. + let Q: Matrix = matrix_new(3, 4) + for i in 0 to 3 { + matrix_set(Q, i, 0, 99.0) + matrix_set(Q, i, 1, (i % 7) as f32) + matrix_set(Q, i, 2, (i % 5) as f32) + matrix_set(Q, i, 3, (i % 3) as f32) + } + + let pred: ptr = random_forest_classifier_predict(m, Q) + for i in 0 to 3 { + if pred[i] != 0.0 && pred[i] != 1.0 { + printf(" FAIL: unseen level produced %.4f, which is not a class label\n", pred[i]) + failed = failed + 1 + } + } + if failed == 0 { + println(" OK: an unseen level predicts a valid label rather than failing") + } + + array_free_f32(pred) + matrix_free(Q) + random_forest_classifier_free(m) + free(cat as ptr) + array_free_f32(y) + matrix_free(X) + return failed +} + +# --------------------------------------------------------------------------- +# 6. The ExtraTree builder takes a mask and produces mask nodes. +# +# ExtraTree draws its split at random, so the assertion is structural rather +# than about accuracy: a categorical feature must produce a subset split, and +# an all-zero mask must produce none. +# --------------------------------------------------------------------------- +function _t457_test_extra_tree_mask() -> i32 { + println("extra tree on a categorical feature") + let mut failed: i32 = 0 + + let reps: i32 = 16 + let n: i32 = 16 * reps + let X: Matrix = _t457_make_x(reps) + let y: ptr = _t457_make_y(reps) + let cat: ptr = _t457_mask_col0(4) + let num: ptr = _t457_mask_none(4) + + let cm: ExtraTreeClassifier = extra_tree_classifier_fit_categorical(X, y, n, 4, 2, 5, 5, cat) + let nm: ExtraTreeClassifier = extra_tree_classifier_fit_categorical(X, y, n, 4, 2, 5, 5, num) + + let mut cmasks: i32 = 0 + for k in 0 to cm.n_nodes { + let nd: ptr = cm.nodes + k + if nd.cat_kind == 1 { cmasks = cmasks + 1 } + } + let mut nmasks: i32 = 0 + for k in 0 to nm.n_nodes { + let nd: ptr = nm.nodes + k + if nd.cat_kind == 1 { nmasks = nmasks + 1 } + } + + if cmasks < 1 { + println(" FAIL: no node of the categorical extra tree carries a mask") + failed = failed + 1 + } else { + printf(" OK: %d mask nodes in the categorical extra tree\n", cmasks) + } + if nmasks != 0 { + printf(" FAIL: the numeric extra tree carries %d mask nodes\n", nmasks) + failed = failed + 1 + } + + # A leaf never carries a left set, whichever way the random split fell. + for k in 0 to cm.n_nodes { + let nd: ptr = cm.nodes + k + if nd.is_leaf && nd.cat_kind != 0 { + printf(" FAIL: leaf node %d carries a left set\n", k) + failed = failed + 1 + } + } + + # The predictions must still be labels the model was trained on. + let pred: ptr = extra_tree_classifier_predict(cm, X) + for i in 0 to n { + if pred[i] != 0.0 && pred[i] != 1.0 { + printf(" FAIL: extra tree predicted %.4f, which is not a class label\n", pred[i]) + failed = failed + 1 + } + } + + array_free_f32(pred) + extra_tree_classifier_free(cm) + extra_tree_classifier_free(nm) + free(cat as ptr) + free(num as ptr) + array_free_f32(y) + matrix_free(X) + return failed +} + +# --------------------------------------------------------------------------- +# 7. The regression side: a bagged regressor on a subset optimum. +# --------------------------------------------------------------------------- +function _t457_test_bagging_regressor() -> i32 { + println("bagging regressor on a categorical feature") + let mut failed: i32 = 0 + + let reps: i32 = 16 + let n: i32 = 16 * reps + let X: Matrix = _t457_make_x(reps) + + # Response is 10 on the even levels and 0 on the odd ones, the same {0,2} + # against {1,3} partition, so the exact fit has residual 0. + let y: ptr = array_new_f32(n) + for i in 0 to n { + let lvl: i32 = i % 16 + let odd: f32 = ((lvl % 2) * 10) as f32 + y[i] = 10.0 - odd + } + + let cat: ptr = _t457_mask_col0(4) + let num: ptr = _t457_mask_none(4) + + let cm: BaggingRegressor = bagging_regressor_fit_categorical(X, y, 8, 1, 13, cat) + let nm: BaggingRegressor = bagging_regressor_fit_categorical(X, y, 8, 1, 13, num) + + let cp: ptr = bagging_regressor_predict(cm, X) + let np: ptr = bagging_regressor_predict(nm, X) + + let mut cres: f32 = 0.0 + let mut nres: f32 = 0.0 + for i in 0 to n { + let cd: f32 = cp[i] - y[i] + let nd: f32 = np[i] - y[i] + cres = cres + cd * cd + nres = nres + nd * nd + } + + if cres >= nres { + printf(" FAIL: categorical residual %.4f did not beat label-encoded %.4f\n", cres, nres) + failed = failed + 1 + } else { + printf(" OK: categorical residual %.4f against label-encoded %.4f\n", cres, nres) + } + + array_free_f32(cp) + array_free_f32(np) + bagging_regressor_free(cm) + bagging_regressor_free(nm) + free(cat as ptr) + free(num as ptr) + array_free_f32(y) + matrix_free(X) + return failed +} + +# --------------------------------------------------------------------------- +# 8. max_features scopes the categorical scan. +# +# A categorical feature that was not drawn at a node must not be scanned there, +# exactly as a numeric one is not. With three features and max_features = 1 the +# informative column is drawn at roughly a third of the nodes, so a forest deep +# enough to have many nodes must contain both mask nodes and threshold nodes. +# A builder that scanned every categorical feature regardless of the draw would +# put a mask on every internal node, since the categorical column is the only +# one that separates the classes at all. +# --------------------------------------------------------------------------- +function _t457_test_max_features_scopes_the_scan() -> i32 { + println("max_features scopes the categorical scan") + let mut failed: i32 = 0 + + let reps: i32 = 16 + let X: Matrix = _t457_make_x(reps) + let y: ptr = _t457_make_y(reps) + let cat: ptr = _t457_mask_col0(4) + + let m: RandomForestClassifier = random_forest_classifier_fit_categorical(X, y, 2, 20, 4, 3, cat) + + let mut internal: i32 = 0 + let mut masked: i32 = 0 + for t in 0 to m.n_trees { + let tree: DecisionTreeClassifier = m.trees[t] + for k in 0 to tree.n_nodes { + let nd: ptr = tree.nodes + k + if not nd.is_leaf { + internal = internal + 1 + if nd.cat_kind == 1 { masked = masked + 1 } + } + } + } + + if masked < 1 { + println(" FAIL: no internal node carries a mask, so the scan never ran") + failed = failed + 1 + } + if masked >= internal { + printf(" FAIL: %d of %d internal nodes are masked, so the scan ignored max_features\n", masked, internal) + failed = failed + 1 + } + if failed == 0 { + printf(" OK: %d of %d internal nodes are subset splits\n", masked, internal) + } + + # Every masked node must actually split on the column that is categorical. + for t in 0 to m.n_trees { + let tree: DecisionTreeClassifier = m.trees[t] + for k in 0 to tree.n_nodes { + let nd: ptr = tree.nodes + k + if nd.cat_kind == 1 && nd.feature != 0 { + printf(" FAIL: a mask node splits on numeric feature %d\n", nd.feature) + failed = failed + 1 + } + } + } + + random_forest_classifier_free(m) + free(cat as ptr) + array_free_f32(y) + matrix_free(X) + return failed +} + +function main() -> i32 { + println("=== Issue #457: categorical splits in ensembles ===") + println("") + + let mut failures: i32 = 0 + failures = failures + _t457_test_forest_beats_label_encoding() + println("") + failures = failures + _t457_test_bagging_beats_label_encoding() + println("") + failures = failures + _t457_test_empty_mask_is_the_plain_fit() + println("") + failures = failures + _t457_test_seed_stability() + println("") + failures = failures + _t457_test_unseen_level_predicts() + println("") + failures = failures + _t457_test_extra_tree_mask() + println("") + failures = failures + _t457_test_bagging_regressor() + println("") + failures = failures + _t457_test_max_features_scopes_the_scan() + println("") + + if failures == 0 { + println("All categorical ensemble tests passed!") + return 0 + } + print("FAILURES: ") + print(failures) + println("") + return 1 +} diff --git a/tests/test_opt_randomforestclassifier_fit.flow b/tests/test_opt_randomforestclassifier_fit.flow index 5cf2a81..4f98080 100644 --- a/tests/test_opt_randomforestclassifier_fit.flow +++ b/tests/test_opt_randomforestclassifier_fit.flow @@ -299,7 +299,13 @@ function rfft_tree_fit(X: Matrix, y: ptr, n_classes: i32, max_depth: i32, c } let max_nodes: i32 = 2 * X.rows + 1 - let nodes: ptr = malloc((max_nodes as i64) * 64) as ptr + # calloc, matching #403 and the builder this copy mirrors. The copy is + # older than #403 and still used malloc, so every field this builder never + # writes held whatever was on the heap: harmless while nothing read them, + # but cat_kind is read from #457 on, and a garbage cat_kind routes the + # reference forest's predictions through a mask that was never fitted. + # 64 bytes per node, above sizeof(TreeNode) = 52; see the struct comment. + let nodes: ptr = calloc(max_nodes as i64, 64) as ptr let n_nodes_ptr: ptr = malloc(4) as ptr n_nodes_ptr[0] = 0