Skip to content

Commit 30d5875

Browse files
youknowoneyouknowone
committed
Implement Py_mod_create slot support in multi-phase init
1 parent 678463d commit 30d5875

54 files changed

Lines changed: 727 additions & 675 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎benches/microbenchmarks.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,10 +113,9 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenc
113113
settings.write_bytecode = false;
114114
settings.user_site_directory = false;
115115

116-
Interpreter::with_init(settings, |vm| {
117-
vm.add_native_module_defs(rustpython_stdlib::get_module_defs());
118-
})
119-
.enter(|vm| {
116+
let builder = Interpreter::builder(settings);
117+
let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx);
118+
builder.add_native_modules(&defs).build().enter(|vm| {
120119
let setup_code = vm
121120
.compile(&bench.setup, Mode::Exec, bench.name.to_owned())
122121
.expect("Error compiling setup code");

‎crates/derive-impl/src/pymodule.rs‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -144,18 +144,6 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To
144144
})
145145
}
146146
},
147-
parse_quote! {
148-
#[allow(dead_code)]
149-
pub(crate) fn make_module(
150-
vm: &::rustpython_vm::VirtualMachine
151-
) -> ::rustpython_vm::PyRef<::rustpython_vm::builtins::PyModule> {
152-
use ::rustpython_vm::PyPayload;
153-
let module = ::rustpython_vm::builtins::PyModule::from_def(module_def(&vm.ctx)).into_ref(&vm.ctx);
154-
__init_dict(vm, &module);
155-
module_exec(vm, &module).unwrap();
156-
module
157-
}
158-
},
159147
]);
160148
}
161149
if !is_submodule && !context.has_module_exec {
@@ -202,7 +190,6 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To
202190
vm: &::rustpython_vm::VirtualMachine,
203191
module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>,
204192
) {
205-
module.__init_methods(vm).unwrap();
206193
__init_attributes(vm, module);
207194
}
208195
},

‎crates/derive/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream {
144144

145145
/// This attribute must be applied to an inline module.
146146
/// It defines a Python module in the form of a `module_def` function in the module;
147-
/// this has to be used in a `get_module_defs` to properly register the module.
147+
/// this has to be used in a `stdlib_module_defs` to properly register the module.
148148
/// Additionally, this macro defines 'MODULE_NAME' and 'DOC' in the module.
149149
/// # Arguments
150150
/// - `name`: the name of the python module,

crates/stdlib/src/opcode.rs renamed to crates/stdlib/src/_opcode.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
pub(crate) use opcode::module_def;
1+
pub(crate) use _opcode::module_def;
22

33
#[pymodule]
4-
mod opcode {
4+
mod _opcode {
55
use crate::vm::{
66
AsObject, PyObjectRef, PyResult, VirtualMachine,
77
builtins::{PyInt, PyIntRef},

‎crates/stdlib/src/gc.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ pub(crate) use gc::module_def;
22

33
#[pymodule]
44
mod gc {
5-
use crate::vm::{PyResult, VirtualMachine, function::FuncArgs};
5+
use crate::vm::{PyObjectRef, PyResult, VirtualMachine, function::FuncArgs};
66

77
#[pyfunction]
88
fn collect(_args: FuncArgs, _vm: &VirtualMachine) -> i32 {
@@ -45,8 +45,10 @@ mod gc {
4545
}
4646

4747
#[pyfunction]
48-
fn get_referrers(_args: FuncArgs, vm: &VirtualMachine) -> PyResult {
49-
Err(vm.new_not_implemented_error(""))
48+
fn get_referrers(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef {
49+
// RustPython does not support bi-directional reference tracking.
50+
// Return empty list for compatibility with reference cycle tests.
51+
vm.ctx.new_list(vec![]).into()
5052
}
5153

5254
#[pyfunction]

‎crates/stdlib/src/lib.rs‎

Lines changed: 100 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,10 @@ mod json;
3838
#[cfg(not(any(target_os = "ios", target_arch = "wasm32")))]
3939
mod locale;
4040

41+
mod _opcode;
4142
mod math;
4243
#[cfg(any(unix, windows))]
4344
mod mmap;
44-
mod opcode;
4545
mod pyexpat;
4646
mod pystruct;
4747
mod random;
@@ -105,131 +105,109 @@ mod tkinter;
105105
use rustpython_common as common;
106106
use rustpython_vm as vm;
107107

108-
use crate::vm::{builtins, stdlib::StdlibDefFunc};
109-
use alloc::borrow::Cow;
108+
use crate::vm::{Context, builtins};
110109

111110
/// Returns module definitions for multi-phase init modules.
112111
/// These modules are added to sys.modules BEFORE their exec function runs,
113112
/// allowing safe circular imports.
114-
pub fn get_module_defs() -> impl Iterator<Item = (Cow<'static, str>, StdlibDefFunc)> {
115-
macro_rules! modules {
116-
{
117-
$(
118-
#[cfg($cfg:meta)]
119-
{ $( $key:expr => $val:expr),* $(,)? }
120-
)*
121-
} => {{
122-
[
123-
$(
124-
$(#[cfg($cfg)] (Cow::<'static, str>::from($key), $val as StdlibDefFunc),)*
125-
)*
126-
]
127-
.into_iter()
128-
}};
113+
pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef> {
114+
let mut defs = vec![
115+
array::module_def(ctx),
116+
binascii::module_def(ctx),
117+
bisect::module_def(ctx),
118+
blake2::module_def(ctx),
119+
bz2::module_def(ctx),
120+
cmath::module_def(ctx),
121+
contextvars::module_def(ctx),
122+
csv::module_def(ctx),
123+
faulthandler::module_def(ctx),
124+
gc::module_def(ctx),
125+
hashlib::module_def(ctx),
126+
json::module_def(ctx),
127+
math::module_def(ctx),
128+
md5::module_def(ctx),
129+
_opcode::module_def(ctx),
130+
random::module_def(ctx),
131+
sha1::module_def(ctx),
132+
sha256::module_def(ctx),
133+
sha3::module_def(ctx),
134+
sha512::module_def(ctx),
135+
statistics::module_def(ctx),
136+
pystruct::module_def(ctx),
137+
suggestions::module_def(ctx),
138+
zlib::module_def(ctx),
139+
pyexpat::module_def(ctx),
140+
unicodedata::module_def(ctx),
141+
];
142+
143+
#[cfg(any(unix, target_os = "wasi"))]
144+
defs.push(fcntl::module_def(ctx));
145+
146+
#[cfg(any(unix, windows, target_os = "wasi"))]
147+
defs.push(select::module_def(ctx));
148+
149+
#[cfg(not(target_arch = "wasm32"))]
150+
{
151+
defs.push(multiprocessing::module_def(ctx));
152+
defs.push(socket::module_def(ctx));
129153
}
130-
modules! {
131-
#[cfg(all())]
132-
{
133-
"array" => array::module_def,
134-
"binascii" => binascii::module_def,
135-
"_bisect" => bisect::module_def,
136-
"_blake2" => blake2::module_def,
137-
"_bz2" => bz2::module_def,
138-
"cmath" => cmath::module_def,
139-
"_contextvars" => contextvars::module_def,
140-
"_csv" => csv::module_def,
141-
"faulthandler" => faulthandler::module_def,
142-
"gc" => gc::module_def,
143-
"_hashlib" => hashlib::module_def,
144-
"_json" => json::module_def,
145-
"math" => math::module_def,
146-
"_md5" => md5::module_def,
147-
"_opcode" => opcode::module_def,
148-
"_random" => random::module_def,
149-
"_sha1" => sha1::module_def,
150-
"_sha256" => sha256::module_def,
151-
"_sha3" => sha3::module_def,
152-
"_sha512" => sha512::module_def,
153-
"_statistics" => statistics::module_def,
154-
"_struct" => pystruct::module_def,
155-
"_suggestions" => suggestions::module_def,
156-
"zlib" => zlib::module_def,
157-
"pyexpat" => pyexpat::module_def,
158-
"unicodedata" => unicodedata::module_def,
159-
}
160-
#[cfg(any(unix, target_os = "wasi"))]
161-
{
162-
"fcntl" => fcntl::module_def,
163-
}
164-
#[cfg(any(unix, windows, target_os = "wasi"))]
165-
{
166-
"select" => select::module_def,
167-
}
168-
#[cfg(not(target_arch = "wasm32"))]
169-
{
170-
"_multiprocessing" => multiprocessing::module_def,
171-
"_socket" => socket::module_def,
172-
}
173-
#[cfg(not(any(target_os = "android", target_arch = "wasm32")))]
174-
{
175-
"_lzma" => lzma::module_def,
176-
}
177-
#[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))]
178-
{
179-
"_sqlite3" => sqlite::module_def,
180-
}
181-
#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-rustls"))]
182-
{
183-
"_ssl" => ssl::module_def,
184-
}
185-
#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))]
186-
{
187-
"_ssl" => openssl::module_def,
188-
}
189-
#[cfg(windows)]
190-
{
191-
"_overlapped" => overlapped::module_def,
192-
}
193-
#[cfg(unix)]
194-
{
195-
"_posixsubprocess" => posixsubprocess::module_def,
196-
}
197-
#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))]
198-
{
199-
"_posixshmem" => posixshmem::module_def,
200-
}
201-
#[cfg(any(unix, windows))]
202-
{
203-
"mmap" => mmap::module_def,
204-
}
205-
#[cfg(all(unix, not(target_os = "redox")))]
206-
{
207-
"syslog" => syslog::module_def,
208-
"resource" => resource::module_def,
209-
}
210-
#[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))]
211-
{
212-
"termios" => termios::module_def,
213-
}
214-
#[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))]
215-
{
216-
"grp" => grp::module_def,
217-
}
218-
#[cfg(target_os = "macos")]
219-
{
220-
"_scproxy" => scproxy::module_def,
221-
}
222-
#[cfg(not(any(target_os = "android", target_os = "ios", target_os = "windows", target_arch = "wasm32", target_os = "redox")))]
223-
{
224-
"_uuid" => uuid::module_def,
225-
}
226-
#[cfg(not(any(target_os = "ios", target_arch = "wasm32")))]
227-
{
228-
"_locale" => locale::module_def,
229-
}
230-
#[cfg(feature = "tkinter")]
231-
{
232-
"_tkinter" => tkinter::module_def,
233-
}
154+
155+
#[cfg(not(any(target_os = "android", target_arch = "wasm32")))]
156+
defs.push(lzma::module_def(ctx));
157+
158+
#[cfg(all(
159+
feature = "sqlite",
160+
not(any(target_os = "android", target_arch = "wasm32"))
161+
))]
162+
defs.push(sqlite::module_def(ctx));
163+
164+
#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-rustls"))]
165+
defs.push(ssl::module_def(ctx));
166+
167+
#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))]
168+
defs.push(openssl::module_def(ctx));
169+
170+
#[cfg(windows)]
171+
defs.push(overlapped::module_def(ctx));
172+
173+
#[cfg(unix)]
174+
defs.push(posixsubprocess::module_def(ctx));
175+
176+
#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))]
177+
defs.push(posixshmem::module_def(ctx));
178+
179+
#[cfg(any(unix, windows))]
180+
defs.push(mmap::module_def(ctx));
181+
182+
#[cfg(all(unix, not(target_os = "redox")))]
183+
{
184+
defs.push(syslog::module_def(ctx));
185+
defs.push(resource::module_def(ctx));
234186
}
187+
188+
#[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))]
189+
defs.push(termios::module_def(ctx));
190+
191+
#[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))]
192+
defs.push(grp::module_def(ctx));
193+
194+
#[cfg(target_os = "macos")]
195+
defs.push(scproxy::module_def(ctx));
196+
197+
#[cfg(not(any(
198+
target_os = "android",
199+
target_os = "ios",
200+
target_os = "windows",
201+
target_arch = "wasm32",
202+
target_os = "redox"
203+
)))]
204+
defs.push(uuid::module_def(ctx));
205+
206+
#[cfg(not(any(target_os = "ios", target_arch = "wasm32")))]
207+
defs.push(locale::module_def(ctx));
208+
209+
#[cfg(feature = "tkinter")]
210+
defs.push(tkinter::module_def(ctx));
211+
212+
defs
235213
}

‎crates/vm/src/import.rs‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,21 @@ pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult {
9898
}
9999

100100
// Try multi-phase init first (preferred for modules that import other modules)
101-
if let Some(def_func) = vm.state.module_defs.get(module_name) {
102-
let def = def_func(&vm.ctx);
103-
104-
// Phase 1: Create module from definition
105-
let module = PyModule::from_def(def).into_ref(&vm.ctx);
106-
107-
// Initialize module dict using proper method
101+
if let Some(&def) = vm.state.module_defs.get(module_name) {
102+
// Phase 1: Create module (use create slot if provided, else default creation)
103+
let module = if let Some(create) = def.slots.create {
104+
// Custom module creation
105+
let spec = vm.ctx.new_str(module_name);
106+
create(vm, spec.as_object(), def)?
107+
} else {
108+
// Default module creation
109+
PyModule::from_def(def).into_ref(&vm.ctx)
110+
};
111+
112+
// Initialize module dict and methods
113+
// Corresponds to PyModule_FromDefAndSpec: md_def, _add_methods_to_object, PyModule_SetDocString
108114
PyModule::__init_dict_from_def(vm, &module);
115+
module.__init_methods(vm)?;
109116

110117
// Add to sys.modules BEFORE exec (critical for circular import handling)
111118
sys_modules.set_item(module_name, module.clone().into(), vm)?;
@@ -118,16 +125,11 @@ pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult {
118125
return Ok(module.into());
119126
}
120127

121-
// Fall back to legacy single-phase init
122-
let make_module_func = vm.state.module_inits.get(module_name).ok_or_else(|| {
123-
vm.new_import_error(
124-
format!("Cannot import builtin module {module_name}"),
125-
vm.ctx.new_str(module_name),
126-
)
127-
})?;
128-
let module = make_module_func(vm);
129-
sys_modules.set_item(module_name, module.as_object().to_owned(), vm)?;
130-
Ok(module.into())
128+
// Module not found in module_defs
129+
Err(vm.new_import_error(
130+
format!("Cannot import builtin module {module_name}"),
131+
vm.ctx.new_str(module_name),
132+
))
131133
}
132134

133135
#[cfg(feature = "rustpython-compiler")]

‎crates/vm/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ pub use self::object::{
9999
AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact,
100100
PyResult, PyWeakRef,
101101
};
102-
pub use self::vm::{Context, Interpreter, Settings, VirtualMachine};
102+
pub use self::vm::{Context, Interpreter, InterpreterBuilder, Settings, VirtualMachine};
103103

104104
pub use rustpython_common as common;
105105
pub use rustpython_compiler_core::{bytecode, frozen};

0 commit comments

Comments
 (0)