Skip to content

[SYSTEMDS-3961] Add Sparsity-based Recompiler Class to Recompilation Routine #2596

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ywcb00 wants to merge 6 commits into from
Closed

[SYSTEMDS-3961] Add Sparsity-based Recompiler Class to Recompilation Routine #2596

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4c93b4f
feat(main/hops/recompile/SparsityDAGRecompiler.java): create class fo…
ywcb00 Aug 20, 2026
af507a4
feat(main/hops/recompile/SparsityDAGRecompiler.java): mimic the spars…
ywcb00 Aug 24, 2026
f07e0a4
chore(main/hops/recompile/SparsityDAGRecompiler.java): create member …
ywcb00 Aug 25, 2026
485471a
chore(main/hops/recompile/SparsityDAGRecompiler.java): print nnz info…
ywcb00 Aug 25, 2026
d5bedce
chore(main/hops/recompile/SparsityDAGRecompiler.java): add license he…
ywcb00 Aug 25, 2026
747d581
chore(**): apply formatting
ywcb00 Aug 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions src/main/java/org/apache/sysds/conf/DMLConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,9 @@ public class DMLConfig
public static final String NATIVE_BLAS = "sysds.native.blas";
public static final String NATIVE_BLAS_DIR = "sysds.native.blas.directory";
public static final String DAG_LINEARIZATION = "sysds.compile.linearization";
public static final String SPARSITY_REWRITES = "sysds.rewrites.sparsity.enabled"; // boolean
public static final String SPARSITY_ESTIMATOR = "sysds.rewrites.sparsity.estimator"; // see EstiamtionUtils.EstimatorType
public static final String SPARSITY_REWRITES = "sysds.sparsity.rewrites.enabled"; // boolean
public static final String SPARSITY_RECOMPILE = "sysds.sparsity.recompile.enabled"; // boolean
Comment thread
ywcb00 marked this conversation as resolved.
public static final String SPARSITY_ESTIMATOR = "sysds.sparsity.estimator"; // see EstiamtionUtils.EstimatorType
public static final String CODEGEN = "sysds.codegen.enabled"; //boolean
public static final String CODEGEN_API = "sysds.codegen.api"; // see SpoofCompiler.API
public static final String CODEGEN_COMPILER = "sysds.codegen.compiler"; //see SpoofCompiler.CompilerType
Expand Down Expand Up @@ -192,6 +193,7 @@ public class DMLConfig
_defaultVals.put(COMPRESSED_TRANSFORMENCODE, "false");
_defaultVals.put(DAG_LINEARIZATION, DagLinearizer.DEPTH_FIRST.name());
_defaultVals.put(SPARSITY_REWRITES, "false");
_defaultVals.put(SPARSITY_RECOMPILE, "false");
_defaultVals.put(SPARSITY_ESTIMATOR, EstimatorType.BASIC_AVG.name());
_defaultVals.put(CODEGEN, "false" );
_defaultVals.put(CODEGEN_API, GeneratorAPI.JAVA.name() );
Expand Down Expand Up @@ -481,7 +483,7 @@ public String getConfigInfo() {
COMPRESSED_LINALG, COMPRESSED_LOSSY, COMPRESSED_VALID_COMPRESSIONS, COMPRESSED_OVERLAPPING,
COMPRESSED_SAMPLING_RATIO, COMPRESSED_SOFT_REFERENCE_COUNT,
COMPRESSED_COCODE, COMPRESSED_TRANSPOSE, COMPRESSED_TRANSFORMENCODE, DAG_LINEARIZATION,
SPARSITY_REWRITES, SPARSITY_ESTIMATOR,
SPARSITY_REWRITES, SPARSITY_RECOMPILE, SPARSITY_ESTIMATOR,
CODEGEN, CODEGEN_API, CODEGEN_COMPILER, CODEGEN_OPTIMIZER, CODEGEN_PLANCACHE, CODEGEN_LITERALS,
STATS_MAX_WRAP_LEN, LINEAGECACHESPILL, COMPILERASSISTED_RW, BUFFERPOOL_LIMIT, MEMORY_MANAGER,
PRINT_GPU_MEMORY_INFO, AVAILABLE_GPUS, SYNCHRONIZE_GPU, EAGER_CUDA_FREE, GPU_RULE_BASED_PLACEMENT,
Expand Down
8 changes: 7 additions & 1 deletion src/main/java/org/apache/sysds/hops/recompile/Recompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.conf.CompilerConfig.ConfigType;
import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.conf.DMLConfig;
import org.apache.sysds.hops.DataGenOp;
import org.apache.sysds.hops.DataOp;
import org.apache.sysds.hops.FunctionOp;
Expand Down Expand Up @@ -387,6 +388,11 @@ else if( !codegen ) {
memo.extract(hops, status);
}

// sparsity-based DAG recompilation if enabled
if(ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.SPARSITY_RECOMPILE)) {
hops = SparsityDAGRecompiler.optimize(hops, ec);
}

Comment thread
ywcb00 marked this conversation as resolved.
// codegen if enabled
if( codegen ) {
//create deep copy for in-place
Expand All @@ -396,7 +402,7 @@ else if( !codegen ) {
hops = SpoofCompiler.optimize(hops,
(status==null || !status.isInitialCodegen()));
}

// set max parallelism constraint to ensure compilation
// incl rewrites does not lose these hop-lop constraints
Hop.resetVisitStatus(hops);
Expand Down
Loading
Loading