Add Julia/Python tests, unify python and Julia interfaces to cpp solver

.gitignore

@@ -1,6 +1,7 @@
 .vscode/
 .claude/
 build/*
-.venv/
 .DS_Store
-julia/examples/Manifest.toml
+julia/examples/Manifest.toml
+python/marble/__pycache__/
+python/.venv/

CMakeLists.txt

@@ -91,26 +91,39 @@ if(MARBLE_BUILD_PYTHON)
         FetchContent_MakeAvailable(pybind11)
     endif()
 
+    # The compiled extension is the private submodule `marble._core`; the pure
+    # Python `marble` package (python/marble/) wraps it with the high-level API.
     pybind11_add_module(marble_python MODULE bindings/python_bindings.cpp)
     target_link_libraries(marble_python PRIVATE marble)
     set_target_properties(marble_python PROPERTIES
-        OUTPUT_NAME            marble
-        LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/python
+        OUTPUT_NAME            _core
+        LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/python/marble
     )
 
-    # .pyi stub generation (for VSCode / Pylance autocomplete)
+    # Stage the pure Python package next to the freshly built extension so that
+    # adding build/python to sys.path is enough to `import marble` from the tree.
+    add_custom_command(TARGET marble_python POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_directory
+                ${CMAKE_SOURCE_DIR}/python/marble ${CMAKE_BINARY_DIR}/python/marble
+        COMMENT "Staging marble package into build/python/marble"
+        VERBATIM
+    )
+
+    # .pyi stub generation for the compiled core (VSCode / Pylance autocomplete)
     if(MARBLE_GENERATE_PYI)
         find_program(PYBIND11_STUBGEN pybind11-stubgen)
         if(PYBIND11_STUBGEN)
             add_custom_command(TARGET marble_python POST_BUILD
-                COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_SOURCE_DIR}/python/typings
                 COMMAND ${CMAKE_COMMAND} -E env
                         PYTHONPATH=${CMAKE_BINARY_DIR}/python
-                        ${PYBIND11_STUBGEN} marble
-                        --output-dir ${CMAKE_SOURCE_DIR}/python/typings
+                        ${PYBIND11_STUBGEN} marble._core
+                        --output-dir ${CMAKE_SOURCE_DIR}/python
                         --numpy-array-remove-parameters
                         --ignore-unresolved-names "^(m|n)$"
-                COMMENT "Generating marble.pyi stubs"
+                COMMAND ${CMAKE_COMMAND} -E copy
+                        ${CMAKE_SOURCE_DIR}/python/marble/_core.pyi
+                        ${CMAKE_BINARY_DIR}/python/marble/_core.pyi
+                COMMENT "Generating marble/_core.pyi stubs"
                 VERBATIM
             )
         else()
@@ -120,7 +133,8 @@ if(MARBLE_BUILD_PYTHON)
         endif()
     endif()
 
-    install(TARGETS marble_python LIBRARY DESTINATION .)
+    # Wheel layout: site-packages/marble/{__init__.py, _core.so}
+    install(TARGETS marble_python LIBRARY DESTINATION marble)
 endif()
 
 # ---------------------------------------------------------------------------

README.md

@@ -1,10 +1,9 @@
 # Marble
 [![CMake Build](https://github.com/MarbleSolver/RCQP/actions/workflows/cmake-build.yml/badge.svg)](https://github.com/MarbleSolver/RCQP/actions/workflows/cmake-build.yml)
+[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://roboticexplorationlab.org/Marble/)
 
 A C++ solver for quadratic programs with linear complementarity constraints, with Python and Julia bindings.
 
-This repository is currently a work-in-progress and will receive some significant clean up. If you want to get started we recommend checking out the Getting Started with [Julia](https://roboticexplorationlab.org/Marble/getting_started/julia/) or [Python](https://roboticexplorationlab.org/Marble/getting_started/python/) pages. We will try to make sure that the interfaces (Julia and Python) do not change, just the internals.
-
 ## Prerequisites
 
 | Dependency | Required for |
@@ -30,7 +29,7 @@ The project uses CMake presets defined in `CMakePresets.json`. Execute the comma
 cmake --preset python
 cmake --build --preset python
 ```
-Output: `build/python/marble.cpython-<version>-<platform>.so`
+Output: `build/python/marble/_core.cpython-<version>-<platform>.so` (the compiled core inside the `marble` package)
 
 ### Julia bindings
 ```bash
@@ -60,11 +59,59 @@ Inside the container:
 - `python` uses the venv at `/opt/venv`; `import marble` works immediately
 - Julia shared library is at `build/lib/libmarble_julia.so`
 
-## Python:  pip install (editable)
 
-For a proper install into your virtual environment:
+## Julia: use from your own environment
+
+The Julia bindings in `julia/` form a package named `Marble`. It is not registered, so
+add it to your environment with `Pkg.develop`.
 
-**1. Install system dependencies** (CMake, Eigen3, and nlohmann-json are still required at build time):
+**1. Build the Julia shared library** (this needs `CxxWrap` available to the Julia that
+CMake uses, see [Julia bindings](#julia-bindings) above):
+```bash
+julia -e 'using Pkg; Pkg.add("CxxWrap")'
+cmake --preset julia
+cmake --build --preset julia
+```
+This writes `build/lib/libmarble_julia.{dylib,so}`, which `Marble.jl` loads on import.
+By default it looks in this repo's `build/lib/`, so an in-repo build needs no extra
+configuration.
+
+**2. Add the package to your environment and install its dependencies:**
+```julia
+using Pkg
+Pkg.develop(path="/path/to/RCQP/julia")
+Pkg.instantiate()      # pulls in CxxWrap and the other dependencies
+```
+
+**3. Use it:** build a solver, set up the problem matrices, then solve. Matrices may be
+dense or sparse, and solver options are passed to `setup!` as keyword arguments.
+```julia
+using Marble
+using LinearAlgebra
+
+# min 1/2 x'x  s.t.  x1 + x2 = 2  ->  x* = [1, 1]
+solver = Marble.Solver()
+Marble.setup!(solver, Matrix(1.0I, 2, 2), [0.0, 0.0]; J_eq = [1.0 1.0], b_eq = [-2.0])
+res = Marble.solve!(solver)
+println(Marble.converged(res), " ", collect(Marble.z(res)))   # true  ~[1.0, 1.0]
+```
+
+If you build or move the shared library elsewhere, point the package at it with a
+Preference, then restart Julia so the new path takes effect:
+```julia
+using Preferences
+set_preferences!("Marble", "libmarble_julia_path" => "/abs/path/to/libmarble_julia")
+```
+The path omits the file extension; `Marble.jl` appends the platform's `.dylib` / `.so`.
+
+
+## Python: use from your own virtual environment
+
+marble is not on PyPI, so install it from a local checkout of this repository. This
+works with any virtual environment, a fresh one or an existing project's.
+
+**1. Install the system build dependencies** (CMake, Eigen3, and nlohmann-json are
+needed at build time):
 ```bash
 # macOS
 brew install cmake eigen nlohmann-json
@@ -73,59 +120,78 @@ brew install cmake eigen nlohmann-json
 sudo apt install cmake libeigen3-dev nlohmann-json3-dev
 ```
 
-**2. Install Python build dependencies into your venv:**
+**2. Create and activate a virtual environment:**
 ```bash
-pip install scikit-build-core pybind11
+python -m venv .venv                 # or activate an existing environment
+source .venv/bin/activate            # Windows: .venv\Scripts\activate
 ```
 
-**3. Install marble:**
+**3. Install marble from the repo's `python/` directory:**
 ```bash
-pip install -e . --no-build-isolation
+pip install /path/to/RCQP/python
 ```
+pip builds the C++ extension in an isolated environment, fetching `scikit-build-core`
+and `pybind11` automatically, and pulls in `numpy` and `scipy`. Nothing else needs to
+be installed first.
 
-After this, `import marble` works from anywhere in that environment.
+After this, `import marble` works wherever that environment is active:
+```python
+import numpy as np
+import marble
 
-## Python:  usage without installing
+# min 1/2 x'x + q'x  ->  x* = -q
+solver = marble.Solver()
+solver.setup(np.eye(2), np.array([1.0, 2.0]))
+print(solver.solve().z)              # [-1. -2.]
+```
+
+**Developing marble itself:** install the build tools into your environment and use an
+editable install so source edits are picked up without reinstalling:
+```bash
+pip install scikit-build-core pybind11
+pip install -e /path/to/RCQP/python --no-build-isolation
+```
+
+## Python: usage without installing
+
+After building (`cmake --build --preset python`), the `marble` package is staged at
+`build/python/marble/`. Add `build/python/` to your path at runtime, then import it:
 
-Add `build/python/` to your path at runtime:
 ```python
 import sys
-from pathlib import Path
 
-# Insert the path to the `build` directory in PYTHONPATH
-path_to_marble = ...
-sys.path.insert(0, str(path_to_marble / "build" / "python"))
+rcqp_root = "/path/to/RCQP"            # your checkout of this repo
+sys.path.insert(0, rcqp_root + "/build/python")
 
 import marble
 ```
 
 ## VSCode: Python autocomplete
 
-Autocomplete and type-checking are driven by the `.pyi` stubs in `typings/`, which are regenerated automatically each time you build `marble_python` (requires `pybind11-stubgen`).
-
-Two config files in this repo wire everything up for VSCode automatically:
+The high-level API in `python/marble/__init__.py` is type-annotated, and the
+compiled core ships a `python/marble/_core.pyi` stub that is regenerated each time
+you build `marble_python` (requires `pybind11-stubgen`). Pylance reads both from the
+`marble` package, so a single setting wires everything up:
 
 ### `.vscode/settings.json`
 ```json
 {
   "python.analysis.extraPaths": ["${workspaceFolder}/build/python"],
-  "python.analysis.stubPath": "${workspaceFolder}/typings",
   "python.analysis.useLibraryCodeForTypes": true
 }
 ```
 
-- `extraPaths`:  tells Pylance where the compiled `.so` lives so it can be imported
-- `stubPath`:  tells Pylance where to find the `.pyi` stub file for type info
+- `extraPaths`:  tells Pylance where the staged `marble` package (with `_core.so` and `_core.pyi`) lives so it can be imported
 
 ### Install the Pylance extension
-Install the **Pylance** extension in VSCode (`ms-python.vscode-pylance`). It picks up the above settings automatically.
+Install the **Pylance** extension in VSCode (`ms-python.vscode-pylance`). It picks up the above setting automatically.
 
 If stubs are stale or missing, rebuild:
 ```bash
 cmake --build --preset python --target marble_python
 ```
 
-If `pybind11-stubgen` is not installed, CMake will warn but the build still succeeds. Ensure `pybind11-stubgen` is installed with:
+If `pybind11-stubgen` is not installed, CMake will warn but the build still succeeds. Install it with:
 ```bash
 pip install pybind11-stubgen
 ```

bindings/python_bindings.cpp

@@ -38,8 +38,8 @@ static Vec arr_to_vec(py::array_t<double> arr) {
     return Eigen::Map<const Vec>(arr.data(), arr.size());
 }
 
-PYBIND11_MODULE(marble, m) {
-    m.doc() = "Marble: constrained optimization solver with complementarity constraints";
+PYBIND11_MODULE(_core, m) {
+    m.doc() = "Marble compiled core: constrained optimization solver with complementarity constraints";
 
     // -------------------------------------------------------------------------
     // Problem
@@ -78,6 +78,10 @@ PYBIND11_MODULE(marble, m) {
         .def_readonly("n_eq",   &Problem::n_eq)
         .def_readonly("n_ineq", &Problem::n_ineq)
         .def_readonly("n_comp", &Problem::n_comp)
+        .def("obj",           [](const Problem& p, const Vec& z) { return p.obj(z); }, py::arg("z"))
+        .def("residual_eq",   [](const Problem& p, const Vec& z) { return p.residual_eq(z); },   py::arg("z"))
+        .def("residual_ineq", [](const Problem& p, const Vec& z) { return p.residual_ineq(z); }, py::arg("z"))
+        .def("residual_comp", [](const Problem& p, const Vec& z) { return p.residual_comp(z); }, py::arg("z"))
         .def_property_readonly("cost_gradient", [](const Problem& p) { return p.cost_gradient; })
         .def_readonly("cost_const", &Problem::cost_const)
         // Constraint matrices (returned as scipy-compatible CSC tuple: rows, cols, colptr, rowval, nzval)
@@ -326,6 +330,8 @@ PYBIND11_MODULE(marble, m) {
              py::arg("niter"))
         .def("get_problem", &Solver::get_problem,
              py::return_value_policy::reference_internal)
+        .def("get_options", [](Solver& s) -> Solver::Options& { return s.options; },
+             py::return_value_policy::reference_internal)
         // Main solve
         .def("solve", [](Solver& s) { return s.solve(); })
         .def("convergence", &Solver::convergence, py::arg("options"))

julia/Project.toml

@@ -25,3 +25,9 @@ NLPModelsJuMP = "0.13.5"
 Preferences = "1.5.2"
 SparseArrays = "1.10.0"
 julia = "1.10"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]

julia/src/Marble.jl

@@ -15,29 +15,63 @@ module Marble
         @initcxx
     end
 
-    function setup!(solver::Marble.Solver,model::Model, ind_cc1, ind_cc2, comp_type; kwargs...)
+    function setup!(solver::Marble.Solver,model::Model, ind_cc1, ind_cc2, cc_type; kwargs...)
         Marble.update_settings!(solver; kwargs...)
-        setup!(solver, MathOptNLPModel(model), ind_cc1, ind_cc2, comp_type; kwargs...)
+        setup!(solver, MathOptNLPModel(model), ind_cc1, ind_cc2, cc_type; kwargs...)
         return nothing
     end
 
-    function setup!(solver::Marble.Solver,nlp::AbstractNLPModel, ind_cc1, ind_cc2, comp_type; kwargs...)
+    function setup!(solver::Marble.Solver,nlp::AbstractNLPModel, ind_cc1, ind_cc2, cc_type; kwargs...)
         Marble.update_settings!(solver; kwargs...)
         opts = Marble.options(solver)
-        data = jump_to_marble(nlp, ind_cc1, ind_cc2, comp_type)
+        data = jump_to_marble(nlp, ind_cc1, ind_cc2, cc_type)
         Marble.set_problem!(solver, data.Q, data.q, data.c0, data.J_eq, data.b_eq, data.J_ineq, data.b_ineq, data.L, data.l, data.R, data.r, opts)
         return nothing
     end
 
-    function setup!(solver::Marble.Solver, Q, q, 
-                    J_eq=zeros(0, size(Q, 1)), b_eq=zeros(0),
-                    J_ineq=zeros(0, size(Q, 1)), b_ineq=zeros(0),
-                    L=zeros(0, size(Q, 1)), l=zeros(0),
-                    R=zeros(0, size(Q, 1)), r=zeros(0); kwargs...)
-                    println(J_eq)
+    function setup!(solver::Marble.Solver, Q::AbstractMatrix, q::AbstractVector, c0::Real=0.0;
+                    J_eq=nothing, b_eq=nothing, J_ineq=nothing, b_ineq=nothing,
+                    L=nothing, l=nothing, R=nothing, r=nothing, kwargs...)
         Marble.update_settings!(solver; kwargs...)
         opts = Marble.options(solver)
-        Marble.set_problem!(solver, Q, q, 0.0, J_eq, b_eq, J_ineq, b_ineq, L, l, R, r, opts)
+
+        n = length(q)
+        J_eq   = isnothing(J_eq)   ? zeros(0, n) : J_eq
+        b_eq   = isnothing(b_eq)   ? zeros(0)    : b_eq
+        J_ineq = isnothing(J_ineq) ? zeros(0, n) : J_ineq
+        b_ineq = isnothing(b_ineq) ? zeros(0)    : b_ineq
+        L = isnothing(L) ? zeros(0, n) : L
+        l = isnothing(l) ? zeros(0)    : l
+        R = isnothing(R) ? zeros(0, n) : R
+        r = isnothing(r) ? zeros(0)    : r
+
+        _set_problem!(solver, opts, Q, q, Float64(c0),
+                      J_eq, b_eq, J_ineq, b_ineq, L, l, R, r)
+        return nothing
+    end
+
+    # Dispatch to the dense or sparse set_problem! binding based on storage. When
+    # any block is sparse every block is converted to a SparseMatrixCSC so the
+    # solver sees consistent compressed-sparse data
+    function _set_problem!(solver::Marble.Solver, opts, Q, q, c0,
+                           J_eq, b_eq, J_ineq, b_ineq, L, l, R, r)
+        blocks = (Q, J_eq, J_ineq, L, R)
+        fvec(v) = collect(Float64, v)
+        if any(b -> b isa AbstractSparseMatrix, blocks)
+            Qs, Es, Is, Ls, Rs = sparse(Q), sparse(J_eq), sparse(J_ineq), sparse(L), sparse(R)
+            Marble.set_problem!(solver, size(Q, 2),
+                Qs.colptr, Qs.rowval, Qs.nzval, fvec(q), c0,
+                length(b_eq),   Es.colptr, Es.rowval, Es.nzval, fvec(b_eq),
+                length(b_ineq), Is.colptr, Is.rowval, Is.nzval, fvec(b_ineq),
+                length(l),      Ls.colptr, Ls.rowval, Ls.nzval, fvec(l),
+                                Rs.colptr, Rs.rowval, Rs.nzval, fvec(r), opts)
+        else
+            fmat(M) = Matrix{Float64}(M)
+            Marble.set_problem!(solver,
+                fmat(Q), fvec(q), c0,
+                fmat(J_eq), fvec(b_eq), fmat(J_ineq), fvec(b_ineq),
+                fmat(L), fvec(l), fmat(R), fvec(r), opts)
+        end
         return nothing
     end