From de264797f934906ed83939bd2a43a3b5db3db35b Mon Sep 17 00:00:00 2001 From: digikar99 Date: Fri, 7 Aug 2020 02:25:14 +0530 Subject: [PATCH] add keyword argument to import-module to import submodules --- src/import-export.lisp | 67 +++++++++++++++++++++++++++++++++--------- tests/tests.lisp | 26 ++++++++++++++++ 2 files changed, 79 insertions(+), 14 deletions(-) diff --git a/src/import-export.lisp b/src/import-export.lisp index 82319f5e..31ebe04f 100644 --- a/src/import-export.lisp +++ b/src/import-export.lisp @@ -43,7 +43,40 @@ module to be imported into the python session. ,(or docstring "Python function") (apply #'python-call ,fun-name args)))) -(defmacro import-module (module-name &key (as module-name as-supplied-p) (reload nil)) +(defvar *is-submodule* nil + "Used for coordinating import statements from defpymodule while calling recursively") + +;;; packages in python are collection of modules; module is a single python file +;;; In fact, all packages are modules; but all modules are not packages. +(defun import-submodules (pymodule-name lisp-package) + (let ((submodules + (python-eval "tuple((modname, ispkg) for importer, modname, ispkg in " + "pkgutil.iter_modules(" + pymodule-name + ".__path__))"))) + (loop :for (submodule has-submodules) :in submodules + :for submodule-fullname := (concatenate 'string + pymodule-name "." submodule) + :if (and (char/= #\_ (aref submodule 0)) ; avoid private modules / packages + ;; pkgutil is of type module + ;; import matplotlib does not import matplotlib.pyplot + ;; https://stackoverflow.com/questions/14812342/matplotlib-has-no-attribute-pyplot + ;; We maintain these semantics. + ;; The below form errors in the case of submodules and + ;; therefore returns NIL. + (ignore-errors (python-eval "type(" submodule-fullname + ") == type(pkgutil)"))) + :collect (let ((*is-submodule* t)) + (macroexpand-1 + `(import-module ,submodule-fullname + :import-submodules ,has-submodules + :as ,(concatenate 'string + pymodule-name "." + submodule))))))) + + +(defmacro import-module (module-name &key (as module-name as-supplied-p) (reload nil) + import-submodules) "Import a python module as a Lisp package. The module name should be a string. @@ -76,14 +109,16 @@ RELOAD specifies that the package should be deleted and reloaded. (python-start-if-not-alive) ;; Import the required module in python - (if as-supplied-p - (python-exec (concatenate 'string - "import " module-name " as " as)) - (python-exec (concatenate 'string - "import " module-name))) + (unless *is-submodule* + (if as-supplied-p + (python-exec (concatenate 'string + "import " module-name " as " as)) + (python-exec (concatenate 'string + "import " module-name)))) ;; Also need to import the "inspect" module (python-exec "import inspect") + (python-exec "import pkgutil") ;; fn-names All callables whose names don't start with "_" (let* ((fn-names (python-eval (concatenate 'string @@ -94,24 +129,28 @@ RELOAD specifies that the package should be deleted and reloaded. ;; so that the result reflects changes to the readtable ;; Setting *package* causes symbols to be interned by READ-FROM-STRING in this package ;; Note that the package doesn't use CL to avoid shadowing - (*package* (make-package (string (read-from-string as)) + (package-name (string (read-from-string as))) + (*package* (make-package package-name :use '())) (fun-symbols (map 'list (lambda (fun-name) (read-from-string fun-name)) fn-names))) - (import '(cl:nil)) ; So that missing docstring is handled + (import '(cl:nil)) ; So that missing docstring is handled `(progn ,(macroexpand `(defpackage ,(package-name *package*) (:use) (:export ,@fun-symbols))) + ,@(if import-submodules + (import-submodules as + package-name)) ,@(loop for name across fn-names - for fn-symbol = (read-from-string name) - for fullname = (concatenate 'string as "." name) ; Include module prefix - append `((import-function ,fullname :as ,fn-symbol - :docstring ,(python-eval (concatenate 'string - as "." name ".__doc__"))) - (export ',fn-symbol ,*package*))) + for fn-symbol = (read-from-string name) + for fullname = (concatenate 'string as "." name) ; Include module prefix + append `((import-function ,fullname :as ,fn-symbol + :docstring ,(python-eval (concatenate 'string + as "." name ".__doc__"))) + (export ',fn-symbol ,*package*))) t))) (defun export-function (function python-name) diff --git a/tests/tests.lisp b/tests/tests.lisp index 54b8a6cb..cafabba4 100644 --- a/tests/tests.lisp +++ b/tests/tests.lisp @@ -1,4 +1,7 @@ (py4cl:import-module "math" :reload t) +(py4cl:import-module "numpy" :reload t :as "NP") +(py4cl:import-module "numpy.random" :import-submodules t :reload t) + (defpackage #:py4cl/tests (:use #:cl #:clunit) (:export #:run)) @@ -709,3 +712,26 @@ class Foo(): (assert-equality #'= #C(0.5 1.0) (py4cl:python-eval #C(1 2) "*" 1/2))) + +;; ==================== SUBMODULE-IMPORT (py4cl2-backport) ========================= + +(deftest numpy-import-as-np (pytests) + ;; also check whether "all" options as expected + (py4cl:import-module "numpy" :as "NP" :reload t) + ;; np. formats are accessible + (assert-true (py4cl:python-eval 'np.float32)) + ;; The below test should pass on py4cl2 but fail on py4cl, due to the presence + ;; of "import statements" in the former + ;; (py4cl:python-stop) + ;;;; package is imported as np even after stopping + ;; (assert-equalp #(5 7 9) (np:add '(1 2 3) '(4 5 6))) + ) + +(deftest numpy-random-import (pytests) + (py4cl:import-module "numpy.random" :reload t :import-submodules t) + ;; The following tests two bugfixes + ;; 1. defpysubmodules was previously importing only packages. + ;; 2. package-import-string was not good for submodules like matplotlib.pyplot + ;; Note also that some symbols are present in pip numpy not in travis apt numpy. + ;; py4cl/tests should not even compile in the case of these bugs. + (assert-true (numpy.random.mtrand:rand 2)))