AP_Scripting: Add support for REPL over MAVLink

libraries/AP_Scripting/AP_Scripting.cpp

@@ -104,8 +104,57 @@ void AP_Scripting::init(void) {
     }
 }
 
+MAV_RESULT AP_Scripting::handle_command_int_packet(const mavlink_command_int_t &packet) {
+    switch ((SCRIPTING_CMD)packet.param1) {
+        case SCRIPTING_CMD_REPL_START:
+            return repl_start() ? MAV_RESULT_ACCEPTED : MAV_RESULT_FAILED;
+        case SCRIPTING_CMD_REPL_STOP:
+            repl_stop();
+            return MAV_RESULT_ACCEPTED;
+        case SCRIPTING_CMD_ENUM_END: // cope with MAVLink generator appending to our enum
+            break;
+    }
+
+    return MAV_RESULT_UNSUPPORTED;
+}
+
+bool AP_Scripting::repl_start(void) {
+    if (terminal.session) { // it's already running, this is fine
+        return true;
+    }
+
+    // nuke the old folder and all contents
+    struct stat st;
+    if ((AP::FS().stat(REPL_DIRECTORY, &st) == -1) &&
+        (AP::FS().unlink(REPL_DIRECTORY)  == -1) &&
+        (errno != EEXIST)) {
+        gcs().send_text(MAV_SEVERITY_INFO, "Unable to delete old REPL %s", strerror(errno));
+    }
+
+    // create a new folder
+    AP::FS().mkdir(REPL_DIRECTORY);
+    // delete old files in case we couldn't
+    AP::FS().unlink(REPL_DIRECTORY "/in");
+    AP::FS().unlink(REPL_DIRECTORY "/out");
+
+    // make the output pointer
+    terminal.output_fd = AP::FS().open(REPL_OUT, O_WRONLY|O_CREAT|O_TRUNC);
+    if (terminal.output_fd == -1) {
+        gcs().send_text(MAV_SEVERITY_INFO, "Unable to make new REPL");
+        return false;
+    }
+
+    terminal.session = true;
+    return true;
+}
+
+void AP_Scripting::repl_stop(void) {
+    terminal.session = false;
+    // can't do any more cleanup here, closing the open FD's is the REPL's responsibility
+}
+
 void AP_Scripting::thread(void) {
-    lua_scripts *lua = new lua_scripts(_script_vm_exec_count, _script_heap_size, _debug_level);
+    lua_scripts *lua = new lua_scripts(_script_vm_exec_count, _script_heap_size, _debug_level, terminal);
     if (lua == nullptr || !lua->heap_allocated()) {
         gcs().send_text(MAV_SEVERITY_CRITICAL, "Unable to allocate scripting memory");
         _init_failed = true;

libraries/AP_Scripting/AP_Scripting.h

@@ -18,6 +18,8 @@
 
 #include <AP_Common/AP_Common.h>
 #include <AP_Param/AP_Param.h>
+#include <GCS_MAVLink/GCS.h>
+#include <AP_Filesystem/AP_Filesystem.h>
 
 class AP_Scripting
 {
@@ -37,7 +39,19 @@ public:
 
     static const struct AP_Param::GroupInfo var_info[];
 
+    MAV_RESULT handle_command_int_packet(const mavlink_command_int_t &packet);
+
+    struct terminal_s {
+        int output_fd;
+        off_t input_offset;
+        bool session;
+    } terminal;
+
 private:
+
+    bool repl_start(void);
+    void repl_stop(void);
+
     void load_script(const char *filename); // load a script from a file
 
     void thread(void); // main script execution thread

libraries/AP_Scripting/lua_repl.cpp

@@ -0,0 +1,261 @@
+// this implements a Lua REPL, and is based off of a cut down version of
+// lua/src/lua.c. It overall modified the functions to the minimum amount
+// required, with the exception of fixing whitespace/indentation on if's
+
+
+#include "lua_scripts.h"
+#include "lua_generated_bindings.h"
+
+#include "lua/src/lua.h"
+#include "lua/src/lauxlib.h"
+#include "lua/src/lualib.h"
+
+#if !defined(LUA_MAXINPUT)
+#define LUA_MAXINPUT    256
+#endif
+
+#if !defined(LUA_PROMPT)
+#define LUA_PROMPT		"> "
+#define LUA_PROMPT2		">> "
+#endif
+
+extern const AP_HAL::HAL& hal;
+
+/*
+** Message handler used to run all chunks
+*/
+static int msghandler(lua_State *L) {
+    const char *msg = lua_tostring(L, 1);
+    if (msg == NULL) {  /* is error object not a string? */
+        if (luaL_callmeta(L, 1, "__tostring") &&  /* does it have a metamethod */
+            lua_type(L, -1) == LUA_TSTRING) { /* that produces a string? */
+            return 1;  /* that is the message */
+        } else {
+            msg = lua_pushfstring(L, "(error object is a %s value)",
+                                     luaL_typename(L, 1));
+        }
+    }
+    luaL_traceback(L, L, msg, 1);  /* append a standard traceback */
+    return 1;  /* return the traceback */
+}
+
+
+/*
+** Interface to 'lua_pcall', which sets appropriate message function
+** and C-signal handler. Used to run all chunks.
+*/
+int lua_scripts::docall(lua_State *L, int narg, int nres) {
+    int status;
+    int base = lua_gettop(L) - narg;  /* function index */
+    lua_rawgeti(L, LUA_REGISTRYINDEX, sandbox_ref);
+    lua_setupvalue(L, -2, 1);
+    lua_pushcfunction(L, msghandler);  /* push message handler */
+    lua_insert(L, base);  /* put it under function and args */
+    status = lua_pcall(L, narg, nres, base);
+    lua_remove(L, base);  /* remove message handler from the stack */
+    return status;
+}
+
+
+/*
+** Returns the string to be used as a prompt by the interpreter.
+*/
+const char * lua_scripts::get_prompt(lua_State *L, int firstline) {
+    const char *p;
+    lua_getglobal(L, firstline ? "_PROMPT" : "_PROMPT2");
+    p = lua_tostring(L, -1);
+    if (p == NULL) {
+        p = (firstline ? LUA_PROMPT : LUA_PROMPT2);
+    }
+    return p;
+}
+
+/* mark in error messages for incomplete statements */
+#define EOFMARK		"<eof>"
+#define marklen		(sizeof(EOFMARK)/sizeof(char) - 1)
+
+
+/*
+** Check whether 'status' signals a syntax error and the error
+** message at the top of the stack ends with the above mark for
+** incomplete statements.
+*/
+int lua_scripts::incomplete(lua_State *L, int status) {
+    if (status == LUA_ERRSYNTAX) {
+        size_t lmsg;
+        const char *msg = lua_tolstring(L, -1, &lmsg);
+        if (lmsg >= marklen && strcmp(msg + lmsg - marklen, EOFMARK) == 0) {
+            lua_pop(L, 1);
+            return 1;
+        }
+    }
+    return 0;  /* else... */
+}
+
+
+/*
+** Prompt the user, read a line, and push it into the Lua stack.
+*/
+int lua_scripts::pushline(lua_State *L, int firstline) {
+    char buffer[LUA_MAXINPUT + 1] = {};
+    ssize_t read_bytes;
+    size_t l = 0;
+
+    // send prompt to the user
+    terminal_print(get_prompt(L, firstline));
+
+    while (terminal.session) {
+        // reseek to where we need input from, as invalid reads could have done weird stuff, and we want to start from the last valid input
+        int input_fd = AP::FS().open(REPL_IN, O_RDONLY);
+        if (input_fd != -1) {
+            AP::FS().lseek(input_fd, terminal.input_offset, SEEK_SET);
+            read_bytes = AP::FS().read(input_fd, buffer, ARRAY_SIZE(buffer) - 1);
+            AP::FS().close(input_fd);
+            if (read_bytes > 0) {
+                // locate the first newline
+                char * newline_chr = strchr(buffer, '\n');
+                if (newline_chr == NULL) {
+                    // we don't have something that looks like a newline, just keep reading till it's longer
+                    read_bytes = 0;
+                } else {
+                    newline_chr[0] = '\0';
+                    // only advance to the newline
+                    l = strlen(buffer);
+                    terminal.input_offset += l + 1;
+                    break;
+                }
+            }
+        }
+        // wait for any input
+        hal.scheduler->delay(100);
+    }
+  
+    lua_pop(L, 1);  /* remove prompt */
+    lua_pushlstring(L, buffer, l);
+    return 1;
+}
+
+
+/*
+** Try to compile line on the stack as 'return <line>;'; on return, stack
+** has either compiled chunk or original line (if compilation failed).
+*/
+int lua_scripts::addreturn(lua_State *L) {
+    const char *line = lua_tostring(L, -1);  /* original line */
+    const char *retline = lua_pushfstring(L, "return %s;", line);
+    int status = luaL_loadbuffer(L, retline, strlen(retline), "=stdin");
+    if (status == LUA_OK) {
+        lua_remove(L, -2);  /* remove modified line */
+    } else {
+        lua_pop(L, 2);  /* pop result from 'luaL_loadbuffer' and modified line */
+    }
+    return status;
+}
+
+
+/*
+** Read multiple lines until a complete Lua statement
+*/
+int lua_scripts::multiline (lua_State *L) {
+    for (;;) {  /* repeat until gets a complete statement */
+        size_t len;
+        const char *line = lua_tolstring(L, 1, &len);  /* get what it has */
+        int status = luaL_loadbuffer(L, line, len, "=stdin");  /* try it */
+        if (!incomplete(L, status) || !pushline(L, 0)) {
+            return status;  /* cannot or should not try to add continuation line */
+        }
+        lua_pushliteral(L, "\n");  /* add newline... */
+        lua_insert(L, -2);  /* ...between the two lines */
+        lua_concat(L, 3);  /* join them */
+    }
+}
+
+
+/*
+** Read a line and try to load (compile) it first as an expression (by
+** adding "return " in front of it) and second as a statement. Return
+** the final status of load/call with the resulting function (if any)
+** in the top of the stack.
+*/
+int lua_scripts::loadline(lua_State *L) {
+    int status;
+    lua_settop(L, 0);
+    if (!pushline(L, 1)) {
+        return -1;  /* no input */
+    }
+    if ((status = addreturn(L)) != LUA_OK) {  /* 'return ...' did not work? */
+        status = multiline(L);  /* try as command, maybe with continuation lines */
+    } else {
+    }
+    lua_remove(L, 1);  /* remove line from the stack */
+    lua_assert(lua_gettop(L) == 1);
+    return status;
+}
+
+// push the tring into the terminal, blocks until it's queued
+void lua_scripts::terminal_print(const char *str) {
+    if ((AP::FS().write(terminal.output_fd, str, strlen(str)) == -1) ||
+        (AP::FS().fsync(terminal.output_fd) != 0)) {
+        terminal.session = false;
+    }
+}
+
+/*
+** Prints (calling the Lua 'print' function) any values on the stack
+*/
+void lua_scripts::l_print(lua_State *L) {
+    int n = lua_gettop(L);
+    if (n > 0) {  /* any result to be printed? */
+        luaL_checkstack(L, LUA_MINSTACK, "too many results to print");
+        // grab all the internal functions via the sandbox
+        lua_rawgeti(L, LUA_REGISTRYINDEX, sandbox_ref);
+        lua_getfield(L, -1, "string");
+        lua_getfield(L, -1, "format");
+        lua_insert(L, 1);
+        lua_remove(L, -2);
+        lua_getfield(L, -1, "rep");
+        lua_remove(L, -2);
+        lua_pushliteral(L, "%s");
+        lua_pushinteger(L, n);
+        lua_pushliteral(L, "\t");
+        if (lua_pcall(L, 3, 1, 0) != LUA_OK) {
+            // should never happen
+            lua_error(L);
+        }
+        lua_insert(L, 2);
+        if (lua_pcall(L, n + 1, 1, 0) != LUA_OK) {
+            terminal_print(lua_pushfstring(L, "error calling 'print' (%s)\n", lua_tostring(L, -1)));
+        } else {
+            terminal_print(lua_pushfstring(L, "%s\n", lua_tostring(L, -1)));
+        }
+    }
+}
+
+/*
+** Do the REPL: repeatedly read (load) a line, evaluate (call) it, and
+** print any results.
+*/
+void lua_scripts::doREPL(lua_State *L) {
+    int status;
+    // clear out any old script results
+    reset_loop_overtime(L);
+    // prep the sandbox
+    create_sandbox(L);
+    sandbox_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+    terminal.input_offset = 0;
+    while (((status = loadline(L)) != -1) && terminal.session) {
+        if (status == LUA_OK) {
+            status = docall(L, 0, LUA_MULTRET);
+        }
+        if (status == LUA_OK) {
+            l_print(L);
+        } else {
+            terminal_print(lua_pushfstring(L, "%s\n", lua_tostring(L, -1)));
+        }
+        reset_loop_overtime(L);
+    }
+    lua_settop(L, 0);  /* clear stack */
+    luaL_unref(L, LUA_REGISTRYINDEX, sandbox_ref);
+    repl_cleanup();
+}
+

libraries/AP_Scripting/lua_scripts.cpp

@@ -33,9 +33,10 @@ extern const AP_HAL::HAL& hal;
 bool lua_scripts::overtime;
 jmp_buf lua_scripts::panic_jmp;
 
-lua_scripts::lua_scripts(const AP_Int32 &vm_steps, const AP_Int32 &heap_size, const AP_Int8 &debug_level)
+lua_scripts::lua_scripts(const AP_Int32 &vm_steps, const AP_Int32 &heap_size, const AP_Int8 &debug_level, struct AP_Scripting::terminal_s &_terminal)
     : _vm_steps(vm_steps),
-      _debug_level(debug_level) {
+      _debug_level(debug_level),
+     terminal(_terminal) {
     _heap = hal.util->allocate_heap_memory(heap_size);
 }
 
@@ -52,6 +53,7 @@ void lua_scripts::hook(lua_State *L, lua_Debug *ar) {
 int lua_scripts::atpanic(lua_State *L) {
     gcs().send_text(MAV_SEVERITY_CRITICAL, "Lua: Panic: %s", lua_tostring(L, -1));
     hal.console->printf("Lua: Panic: %s\n", lua_tostring(L, -1));
+    printf("Lua: Panic: %s\n", lua_tostring(L, -1));
     longjmp(panic_jmp, 1);
     return 0;
 }
@@ -91,8 +93,16 @@ lua_scripts::script_info *lua_scripts::load_script(lua_State *L, char *filename)
     new_script->name = filename;
     new_script->next = nullptr;
 
+    create_sandbox(L);
+    lua_setupvalue(L, -2, 1);
 
-    // find and create a sandbox for the new chunk
+    new_script->lua_ref = luaL_ref(L, LUA_REGISTRYINDEX);   // cache the reference
+    new_script->next_run_ms = AP_HAL::millis64() - 1; // force the script to be stale
+
+    return new_script;
+}
+
+void lua_scripts::create_sandbox(lua_State *L) {
     lua_newtable(L);
     luaopen_base_sandbox(L);
     lua_pushstring(L, "math");
@@ -112,12 +122,7 @@ lua_scripts::script_info *lua_scripts::load_script(lua_State *L, char *filename)
     lua_settable(L, -3);
     load_lua_bindings(L);
     load_generated_sandbox(L);
-    lua_setupvalue(L, -2, 1);
 
-    new_script->lua_ref = luaL_ref(L, LUA_REGISTRYINDEX);   // cache the reference
-    new_script->next_run_ms = AP_HAL::millis64() - 1; // force the script to be stale
-
-    return new_script;
 }
 
 void lua_scripts::load_all_scripts_in_dir(lua_State *L, const char *dirname) {
@@ -164,6 +169,13 @@ void lua_scripts::load_all_scripts_in_dir(lua_State *L, const char *dirname) {
     AP::FS().closedir(d);
 }
 
+void lua_scripts::reset_loop_overtime(lua_State *L) {
+    overtime = false;
+    // reset the hook to clear the counter
+    const int32_t vm_steps = MAX(_vm_steps, 1000);
+    lua_sethook(L, hook, LUA_MASKCOUNT, vm_steps);
+}
+
 void lua_scripts::run_next_script(lua_State *L) {
     if (scripts == nullptr) {
 #if defined(AP_SCRIPTING_CHECKS) && AP_SCRIPTING_CHECKS >= 1
@@ -172,16 +184,12 @@ void lua_scripts::run_next_script(lua_State *L) {
         return;
     }
 
-    // reset the current script tracking information
-    overtime = false;
-
     // strip the selected script out of the list
     script_info *script = scripts;
     scripts = script->next;
 
     // reset the hook to clear the counter
-    const int32_t vm_steps = MAX(_vm_steps, 1000);
+    reset_loop_overtime(L);
-    lua_sethook(L, hook, LUA_MASKCOUNT, vm_steps);
 
     // store top of stack so we can calculate the number of return values
     int stack_top = lua_gettop(L);
@@ -192,7 +200,7 @@ void lua_scripts::run_next_script(lua_State *L) {
     if(lua_pcall(L, 0, LUA_MULTRET, 0)) {
         if (overtime) {
             // script has consumed an excessive amount of CPU time
-            gcs().send_text(MAV_SEVERITY_CRITICAL, "Lua: %s exceeded time limit (%d)", script->name,  (int)vm_steps);
+            gcs().send_text(MAV_SEVERITY_CRITICAL, "Lua: %s exceeded time limit", script->name);
             remove_script(L, script);
         } else {
             hal.console->printf("Lua: Error: %s\n", lua_tostring(L, -1));
@@ -314,6 +322,19 @@ void *lua_scripts::alloc(void *ud, void *ptr, size_t osize, size_t nsize) {
     return hal.util->heap_realloc(_heap, ptr, nsize);
 }
 
+void lua_scripts::repl_cleanup (void) {
+    if (terminal.session) {
+        terminal.session = false;
+        if (terminal.output_fd != -1) {
+            AP::FS().close(terminal.output_fd);
+            terminal.output_fd = -1;
+            AP::FS().unlink(REPL_DIRECTORY "/in");
+            AP::FS().unlink(REPL_DIRECTORY "/out");
+            AP::FS().unlink(REPL_DIRECTORY);
+        }
+    }
+}
+
 void lua_scripts::run(void) {
     bool succeeded_initial_load = false;
 
@@ -336,6 +357,8 @@ void lua_scripts::run(void) {
         }
         scripts = nullptr;
         overtime = false;
+        // end any open REPL sessions
+        repl_cleanup();
     }
 
     lua_state = lua_newstate(alloc, NULL);
@@ -355,6 +378,12 @@ void lua_scripts::run(void) {
     succeeded_initial_load = true;
 
     while (AP_Scripting::get_singleton()->enabled()) {
+        // handle terminal data if we have any
+        if (terminal.session) {
+            doREPL(L);
+            continue;
+        }
+
 #if defined(AP_SCRIPTING_CHECKS) && AP_SCRIPTING_CHECKS >= 1
         if (lua_gettop(L) != 0) {
             AP_HAL::panic("Lua: Stack should be empty before running scripts");

libraries/AP_Scripting/lua_scripts.h

@@ -20,11 +20,28 @@
 
 #include <AP_Filesystem/posix_compat.h>
 #include "lua_bindings.h"
+#include <AP_Scripting/AP_Scripting.h>
+
+#ifndef REPL_DIRECTORY
+  #if HAL_OS_FATFS_IO
+    #define REPL_DIRECTORY "/APM/repl"
+  #else
+    #define REPL_DIRECTORY "./repl"
+  #endif //HAL_OS_FATFS_IO
+#endif // REPL_DIRECTORY
+
+#ifndef REPL_IN
+  #define REPL_IN REPL_DIRECTORY "/in"
+#endif // REPL_IN
+
+#ifndef REPL_OUT
+  #define REPL_OUT REPL_DIRECTORY "/out"
+#endif // REPL_OUT
 
 class lua_scripts
 {
 public:
-    lua_scripts(const AP_Int32 &vm_steps, const AP_Int32 &heap_size, const AP_Int8 &debug_level);
+    lua_scripts(const AP_Int32 &vm_steps, const AP_Int32 &heap_size, const AP_Int8 &debug_level, struct AP_Scripting::terminal_s &_terminal);
 
     /* Do not allow copies */
     lua_scripts(const lua_scripts &other) = delete;
@@ -39,6 +56,10 @@ public:
     static bool overtime; // script exceeded it's execution slot, and we are bailing out
 private:
 
+    void create_sandbox(lua_State *L);
+
+    void repl_cleanup(void);
+
     typedef struct script_info {
        int lua_ref;          // reference to the loaded script object
        uint64_t next_run_ms; // time (in milliseconds) the script should next be run at
@@ -48,6 +69,8 @@ private:
 
     script_info *load_script(lua_State *L, char *filename);
 
+    void reset_loop_overtime(lua_State *L);
+
     void load_all_scripts_in_dir(lua_State *L, const char *dirname);
 
     void run_next_script(lua_State *L);
@@ -57,6 +80,20 @@ private:
     // reschedule the script for execution. It is assumed the script is not in the list already
     void reschedule_script(script_info *script);
 
+    // REPL stuff
+    struct AP_Scripting::terminal_s &terminal;
+    void doREPL(lua_State *L);
+    void l_print(lua_State *L);
+    void terminal_print(const char *str);
+    int loadline(lua_State *L);
+    int multiline(lua_State *L);
+    int addreturn(lua_State *L);
+    int pushline(lua_State *L, int firstline);
+    int incomplete(lua_State *L, int status);
+    const char * get_prompt(lua_State *L, int firstline);
+    int docall(lua_State *L, int narg, int nres);
+    int sandbox_ref;
+
     script_info *scripts; // linked list of scripts to be run, sorted by next run time (soonest first)
 
     // hook will be run when CPU time for a script is exceeded