bpo-37363: Add audit events on startup for the run commands (GH-14524)

Doc/library/sys.rst

@@ -905,6 +905,12 @@ always available.
    read, so that you can set this hook there.  The :mod:`site` module
    :ref:`sets this <rlcompleter-config>`.
 
+   .. audit-event:: cpython.run_interactivehook hook sys.__interactivehook__
+
+      Raises an :ref:`auditing event <auditing>`
+      ``cpython.run_interactivehook`` with the hook object as the argument when
+      the hook is called on startup.
+
    .. versionadded:: 3.4
 
 

Doc/tools/extensions/pyspecific.py

@@ -199,13 +199,18 @@ class AuditEvent(Directive):
                     .format(name, info['args'], new_info['args'])
                 )
 
-        if len(self.arguments) >= 3 and self.arguments[2]:
-            target = self.arguments[2]
         ids = []
-        else:
-            target = "audit_event_{}_{}".format(name, len(info['source']))
-            target = re.sub(r'\W', '_', label)
-            ids = [target]
+        try:
+            target = self.arguments[2].strip("\"'")
+        except (IndexError, TypeError):
+            target = None
+        if not target:
+            target = "audit_event_{}_{}".format(
+                re.sub(r'\W', '_', name),
+                len(info['source']),
+            )
+            ids.append(target)
+
         info['source'].append((env.docname, target))
 
         pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids)
@@ -560,7 +565,8 @@ def process_audit_events(app, doctree, fromdocname):
         row += nodes.entry('', node)
 
         node = nodes.paragraph()
-        for i, (doc, label) in enumerate(audit_event['source'], start=1):
+        backlinks = enumerate(sorted(set(audit_event['source'])), start=1)
+        for i, (doc, label) in backlinks:
             if isinstance(label, str):
                 ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True)
                 ref['refuri'] = "{}#{}".format(

Doc/using/cmdline.rst

@@ -70,6 +70,7 @@ source.
    :data:`sys.path` (allowing modules in that directory to be imported as top
    level modules).
 
+   .. audit-event:: cpython.run_command command cmdoption-c
 
 .. cmdoption:: -m <module-name>
 
@@ -106,13 +107,14 @@ source.
        python -mtimeit -s 'setup here' 'benchmarked code here'
        python -mtimeit -h # for details
 
+   .. audit-event:: cpython.run_module module-name cmdoption-m
+
    .. seealso::
       :func:`runpy.run_module`
          Equivalent functionality directly available to Python code
 
       :pep:`338` -- Executing modules as scripts
 
-
    .. versionchanged:: 3.1
       Supply the package name to run a ``__main__`` submodule.
 
@@ -129,6 +131,7 @@ source.
    ``"-"`` and the current directory will be added to the start of
    :data:`sys.path`.
 
+   .. audit-event:: cpython.run_stdin "" ""
 
 .. describe:: <script>
 
@@ -148,6 +151,8 @@ source.
    added to the start of :data:`sys.path` and the ``__main__.py`` file in
    that location is executed as the :mod:`__main__` module.
 
+   .. audit-event:: cpython.run_file filename
+
    .. seealso::
       :func:`runpy.run_path`
          Equivalent functionality directly available to Python code
@@ -540,6 +545,11 @@ conflict.
    the interactive session.  You can also change the prompts :data:`sys.ps1` and
    :data:`sys.ps2` and the hook :data:`sys.__interactivehook__` in this file.
 
+   .. audit-event:: cpython.run_startup filename PYTHONSTARTUP
+
+      Raises an :ref:`auditing event <auditing>` ``cpython.run_startup`` with
+      the filename as the argument when called on startup.
+
 
 .. envvar:: PYTHONOPTIMIZE
 

Lib/test/test_embed.py

@@ -57,7 +57,8 @@ class EmbeddingTestsMixin:
     def tearDown(self):
         os.chdir(self.oldcwd)
 
-    def run_embedded_interpreter(self, *args, env=None):
+    def run_embedded_interpreter(self, *args, env=None,
+                                 timeout=None, returncode=0, input=None):
         """Runs a test in the embedded interpreter"""
         cmd = [self.test_exe]
         cmd.extend(args)
@@ -73,18 +74,18 @@ class EmbeddingTestsMixin:
                              universal_newlines=True,
                              env=env)
         try:
-            (out, err) = p.communicate()
+            (out, err) = p.communicate(input=input, timeout=timeout)
         except:
             p.terminate()
             p.wait()
             raise
-        if p.returncode != 0 and support.verbose:
+        if p.returncode != returncode and support.verbose:
             print(f"--- {cmd} failed ---")
             print(f"stdout:\n{out}")
             print(f"stderr:\n{err}")
             print(f"------")
 
-        self.assertEqual(p.returncode, 0,
+        self.assertEqual(p.returncode, returncode,
                          "bad returncode %d, stderr is %r" %
                          (p.returncode, err))
         return out, err
@@ -955,6 +956,37 @@ class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_audit_subinterpreter(self):
         self.run_embedded_interpreter("test_audit_subinterpreter")
 
+    def test_audit_run_command(self):
+        self.run_embedded_interpreter("test_audit_run_command", timeout=3, returncode=1)
+
+    def test_audit_run_file(self):
+        self.run_embedded_interpreter("test_audit_run_file", timeout=3, returncode=1)
+
+    def test_audit_run_interactivehook(self):
+        startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
+        with open(startup, "w", encoding="utf-8") as f:
+            print("import sys", file=f)
+            print("sys.__interactivehook__ = lambda: None", file=f)
+        try:
+            env = {**remove_python_envvars(), "PYTHONSTARTUP": startup}
+            self.run_embedded_interpreter("test_audit_run_interactivehook", timeout=5,
+                                          returncode=10, env=env)
+        finally:
+            os.unlink(startup)
+
+    def test_audit_run_startup(self):
+        startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
+        with open(startup, "w", encoding="utf-8") as f:
+            print("pass", file=f)
+        try:
+            env = {**remove_python_envvars(), "PYTHONSTARTUP": startup}
+            self.run_embedded_interpreter("test_audit_run_startup", timeout=5,
+                                          returncode=10, env=env)
+        finally:
+            os.unlink(startup)
+
+    def test_audit_run_stdin(self):
+        self.run_embedded_interpreter("test_audit_run_stdin", timeout=3, returncode=1)
 
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Security/2019-07-01-10-31-14.bpo-37363.fSjatj.rst

@@ -0,0 +1,2 @@
+Adds audit events for the range of supported run commands (see
+:ref:`using-on-general`).

Modules/main.c

@@ -247,6 +247,10 @@ pymain_run_command(wchar_t *command, PyCompilerFlags *cf)
         goto error;
     }
 
+    if (PySys_Audit("cpython.run_command", "O", unicode) < 0) {
+        return pymain_exit_err_print();
+    }
+
     bytes = PyUnicode_AsUTF8String(unicode);
     Py_DECREF(unicode);
     if (bytes == NULL) {
@@ -267,6 +271,9 @@ static int
 pymain_run_module(const wchar_t *modname, int set_argv0)
 {
     PyObject *module, *runpy, *runmodule, *runargs, *result;
+    if (PySys_Audit("cpython.run_module", "u", modname) < 0) {
+        return pymain_exit_err_print();
+    }
     runpy = PyImport_ImportModule("runpy");
     if (runpy == NULL) {
         fprintf(stderr, "Could not import runpy module\n");
@@ -311,6 +318,9 @@ static int
 pymain_run_file(PyConfig *config, PyCompilerFlags *cf)
 {
     const wchar_t *filename = config->run_filename;
+    if (PySys_Audit("cpython.run_file", "u", filename) < 0) {
+        return pymain_exit_err_print();
+    }
     FILE *fp = _Py_wfopen(filename, L"rb");
     if (fp == NULL) {
         char *cfilename_buffer;
@@ -383,6 +393,9 @@ pymain_run_startup(PyConfig *config, PyCompilerFlags *cf, int *exitcode)
     if (startup == NULL) {
         return 0;
     }
+    if (PySys_Audit("cpython.run_startup", "s", startup) < 0) {
+        return pymain_err_print(exitcode);
+    }
 
     FILE *fp = _Py_fopen(startup, "r");
     if (fp == NULL) {
@@ -420,6 +433,10 @@ pymain_run_interactive_hook(int *exitcode)
         return 0;
     }
 
+    if (PySys_Audit("cpython.run_interactivehook", "O", hook) < 0) {
+        goto error;
+    }
+
     result = _PyObject_CallNoArg(hook);
     Py_DECREF(hook);
     if (result == NULL) {
@@ -457,6 +474,10 @@ pymain_run_stdin(PyConfig *config, PyCompilerFlags *cf)
         return pymain_exit_err_print();
     }
 
+    if (PySys_Audit("cpython.run_stdin", NULL) < 0) {
+        return pymain_exit_err_print();
+    }
+
     int run = PyRun_AnyFileExFlags(stdin, "<stdin>", 0, cf);
     return (run != 0);
 }

Programs/_testembed.c

@@ -1235,6 +1235,101 @@ static int test_audit_subinterpreter(void)
     }
 }
 
+typedef struct {
+    const char* expected;
+    int exit;
+} AuditRunCommandTest;
+
+static int _audit_hook_run(const char *eventName, PyObject *args, void *userData)
+{
+    AuditRunCommandTest *test = (AuditRunCommandTest*)userData;
+    if (strcmp(eventName, test->expected)) {
+        return 0;
+    }
+
+    if (test->exit) {
+        PyObject *msg = PyUnicode_FromFormat("detected %s(%R)", eventName, args);
+        if (msg) {
+            printf("%s\n", PyUnicode_AsUTF8(msg));
+            Py_DECREF(msg);
+        }
+        exit(test->exit);
+    }
+
+    PyErr_Format(PyExc_RuntimeError, "detected %s(%R)", eventName, args);
+    return -1;
+}
+
+static int test_audit_run_command(void)
+{
+    AuditRunCommandTest test = {"cpython.run_command"};
+    wchar_t *argv[] = {L"./_testembed", L"-c", L"pass"};
+
+    Py_IgnoreEnvironmentFlag = 0;
+    PySys_AddAuditHook(_audit_hook_run, (void*)&test);
+
+    return Py_Main(Py_ARRAY_LENGTH(argv), argv);
+}
+
+static int test_audit_run_file(void)
+{
+    AuditRunCommandTest test = {"cpython.run_file"};
+    wchar_t *argv[] = {L"./_testembed", L"filename.py"};
+
+    Py_IgnoreEnvironmentFlag = 0;
+    PySys_AddAuditHook(_audit_hook_run, (void*)&test);
+
+    return Py_Main(Py_ARRAY_LENGTH(argv), argv);
+}
+
+static int run_audit_run_test(int argc, wchar_t **argv, void *test)
+{
+    PyStatus status;
+    PyConfig config;
+    status = PyConfig_InitPythonConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+    config.argv.length = argc;
+    config.argv.items = argv;
+    config.parse_argv = 1;
+    config.program_name = argv[0];
+    config.interactive = 1;
+    config.isolated = 0;
+    config.use_environment = 1;
+    config.quiet = 1;
+
+    PySys_AddAuditHook(_audit_hook_run, test);
+
+    status = Py_InitializeFromConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    return Py_RunMain();
+}
+
+static int test_audit_run_interactivehook(void)
+{
+    AuditRunCommandTest test = {"cpython.run_interactivehook", 10};
+    wchar_t *argv[] = {L"./_testembed"};
+    return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
+}
+
+static int test_audit_run_startup(void)
+{
+    AuditRunCommandTest test = {"cpython.run_startup", 10};
+    wchar_t *argv[] = {L"./_testembed"};
+    return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
+}
+
+static int test_audit_run_stdin(void)
+{
+    AuditRunCommandTest test = {"cpython.run_stdin"};
+    wchar_t *argv[] = {L"./_testembed"};
+    return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
+}
+
 static int test_init_read_set(void)
 {
     PyStatus status;
@@ -1413,6 +1508,11 @@ static struct TestCase TestCases[] = {
     {"test_open_code_hook", test_open_code_hook},
     {"test_audit", test_audit},
     {"test_audit_subinterpreter", test_audit_subinterpreter},
+    {"test_audit_run_command", test_audit_run_command},
+    {"test_audit_run_file", test_audit_run_file},
+    {"test_audit_run_interactivehook", test_audit_run_interactivehook},
+    {"test_audit_run_startup", test_audit_run_startup},
+    {"test_audit_run_stdin", test_audit_run_stdin},
     {NULL, NULL}
 };